per_node_diag = [{'ns_1@10.1.5.178', [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {manifest, ["bucket_engine_1.6.5_10_g3c82cbf-Linux.i686.tar.gz", "curl-7.21.1-w64_patched.tar.gz", "ep-engine_1.6.5.3_143_g366c22e-Linux.i686.tar.gz", "google-perftools-1.7","google-perftools-1.7.tar.gz", "libconflate_1.6.5_1_g8a42194-Linux.i686.tar.gz", "libevent-2.0.7-rc.tar.gz", "libmemcached-0.41_trond-norbye_mingw32-revno895.tar.gz", "libvbucket_1.6.4_1_g5cb588e-Linux.i686.tar.gz", "membase-cli_1.6.5_2_gb9c8a7d-Linux.i686.tar.gz", "memcached_1.4.4_430_g6235145-Linux.i686.tar.gz", "moxi_1.6.5.1_16_g8128746-Linux.i686.tar.gz", "ns_server_1.6.5.3_291_gba5fe27.tar.gz", "portsigar_-Linux.i686.tar.gz", "sigar_sigar_1.6.4_402_g7809620-Linux.i686.tar.gz", "vbucketmigrator_1.7.0r_4_g8428c29-Linux.i686.tar.gz", "wallace_1.7.0r-100-g85ee8f5-Linux.i686-201104060841"]}, {config, [{alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,'filtered-out'}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up, server_joined,server_left,bucket_created, bucket_deleted,bucket_auth_failed]}]}, {buckets, [{'_vclock',[{'ns_1@10.1.5.192',{11,63469523235}}]}, {configs, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers, ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178']}, {map, [['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}]}]}, {memory_quota,1570}, {nodes_wanted, [{'_vclock',[{'ns_1@10.1.5.192',{2,63469523039}}]}, 'ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193']}, {otp, [{'_vclock',[{'ns_1@10.1.5.193',{1,63469523027}}]}, {cookie,lulxzmudqfmlvwvs}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {rebalance_status, {none, <<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>}}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds, [{'_vclock',[{'ns_1@10.1.5.192',{1,63469522657}}]}, {creds, [{"Administrator",[{password,'filtered-out'}]}]}]}, {{node,'ns_1@10.1.5.178',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.178',membership},inactiveFailed}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.178',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.192',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.192',membership},active}, {{node,'ns_1@10.1.5.192',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.192',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.192',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.192',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.193',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.193',membership},inactiveFailed}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.193',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}]}, {basic_info, [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,1120}, {memory_data,{2075373568,1765871616,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41}, {"/boot",101086,13}, {"/dev/shm",1013364,0}]}]}, {processes, [{<0.0.0>, [{registered_name,init}, {status,waiting}, {initial_call,{otp_ring0,start,2}}, {backtrace, ["Program counter: 0xb7c79408 (init:loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb716f79c Return addr 0x081f9e74 ()", "y(0) {state,[{'-root',[<<37 bytes>>]},{'-progname',[<<3 bytes>>]},{'-home',[<<12 bytes"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,88}]}, {heap_size,4181}, {total_heap_size,10946}, {links,[<0.6.0>,<0.7.0>,<0.3.0>]}, {memory,44164}, {message_queue_len,0}, {reductions,47288}, {trap_exit,true}]}, {<0.3.0>, [{registered_name,erl_prim_loader}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7e7e860 (erl_prim_loader:loop/3 + 92)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6fc5bfc Return addr 0x081f9e74 ()", "y(0) []", "y(1) [\"/opt/membase/1.7.0r/bin/ns_server/deps/menelaus/deps/mochiweb/ebin\",\"/opt/memba", "y(2) <0.2.0>", "y(3) {state,efile,[],none,#Port<0.2>,infinity,undefined,true,{prim_state,false,undefin", "y(4) infinity"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,267}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[#Port<0.2>,<0.0.0>]}, {memory,54480}, {message_queue_len,0}, {reductions,888171}, {trap_exit,true}]}, {<0.6.0>, [{registered_name,error_logger}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb714644c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,ns_log_mf_h,false,{state,\"/var/membase/logs\",10485760,10,619247,1,{file", "y(3) error_logger","y(4) <0.2.0>", "0xb7146464 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,37}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<0.0.0>,<0.25.0>,#Port<0.965>]}, {memory,33888}, {message_queue_len,0}, {reductions,222103}, {trap_exit,true}]}, {<0.7.0>, [{registered_name,application_controller}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d640f8 Return addr 0x081f9e74 ()", "y(0) []","y(1) infinity", "y(2) application_controller", "y(3) {state,[],[],[],[{ns_server,<0.52.0>},{mnesia,<0.70.0>},{os_mon,<0.41.0>},{sasl,<", "y(4) application_controller", "y(5) <0.2.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,4181}, {total_heap_size,21892}, {links, [<0.41.0>,<0.52.0>,<0.70.0>,<0.9.0>,<0.32.0>,<0.0.0>]}, {memory,88068}, {message_queue_len,0}, {reductions,31293}, {trap_exit,true}]}, {<0.9.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce8324 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.10.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_ser", "y(2) <0.7.0>", "0xb7ce8334 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.7.0>,<0.10.0>]}, {memory,3436}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<0.10.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ceb2e0 Return addr 0x081f9e74 ()", "y(0) []","y(1) kernel","y(2) <0.11.0>", "y(3) <0.9.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.9.0>,<0.11.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,72}, {trap_exit,true}]}, {<0.11.0>, [{registered_name,kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d07ea0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_sup},one_for_all,[{child,<0.58.0>,net_sup_dynamic,{erl_distr", "y(4) kernel_sup","y(5) <0.10.0>", "0xb7d07ebc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<0.22.0>,<0.26.0>,<0.27.0>,<0.58.0>,<0.24.0>, <0.17.0>,<0.20.0>,<0.21.0>,<0.19.0>,<0.12.0>, <0.13.0>,<0.10.0>]}, {memory,6076}, {message_queue_len,0}, {reductions,1451}, {trap_exit,true}]}, {<0.12.0>, [{registered_name,rex}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6cbbcfc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) rpc", "y(3) {1,{<0.1167.0>,{<0.1166.0>,{#Ref<0.0.0.23940>,'ns_1@10.1.5.178'}},nil,nil}}", "y(4) rex","y(5) <0.11.0>", "0xb6cbbd18 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,196418}, {total_heap_size,514229}, {links,[<0.11.0>]}, {memory,2057396}, {message_queue_len,0}, {reductions,4813}, {trap_exit,true}]}, {<0.13.0>, [{registered_name,global_name_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7772fb8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) global", "y(3) {state,true,['ns_1@10.1.5.192','ns_1@10.1.5.193'],['ns_1@10.1.5.193','ns_1@10.1.5", "y(4) global_name_server","y(5) <0.11.0>", "0xb7772fd4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<0.14.0>,<0.16.0>,<0.15.0>,<0.11.0>]}, {memory,6068}, {message_queue_len,0}, {reductions,1945}, {trap_exit,true}]}, {<0.14.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7f31094 (global:loop_the_locker/1 + 588)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cec254 Return addr 0xb7f30e28 (global:init_the_locker/1 + 192)", "y(0) {multi,[],[],['ns_1@10.1.5.192','ns_1@10.1.5.193'],'ns_1@10.1.5.193',false,false}", "y(1) infinity", "0xb7cec260 Return addr 0x081f9e74 ()"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<0.13.0>]}, {memory,5796}, {message_queue_len,0}, {reductions,547}, {trap_exit,true}]}, {<0.15.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7f35140 (global:collect_deletions/2 + 76)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf1ee4 Return addr 0xb7f350c0 (global:loop_the_deleter/1 + 36)", "y(0) infinity","y(1) []", "y(2) <0.13.0>", "0xb7cf1ef4 Return addr 0x081f9e74 ()", "y(0) <0.13.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.13.0>]}, {memory,3356}, {message_queue_len,0}, {reductions,227}, {trap_exit,false}]}, {<0.16.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7f35260 (global:loop_the_registrar/0 + 12)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d59d58 Return addr 0x081f9e74 ()", "y(0) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.13.0>]}, {memory,4288}, {message_queue_len,0}, {reductions,243}, {trap_exit,false}]}, {<0.17.0>, [{registered_name,inet_db}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10158 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) inet_db", "y(3) {state,inet_db,inet_cache,inet_hosts_byname,inet_hosts_byaddr,inet_hosts_file_byn", "y(4) inet_db","y(5) <0.11.0>", "0xb7d10174 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<0.11.0>]}, {memory,10736}, {message_queue_len,0}, {reductions,1444}, {trap_exit,true}]}, {<0.19.0>, [{registered_name,global_group}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce7748 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) global_group", "y(3) {state,no_conf,true,[],[],[],[],[],'nonode@nohost',[],normal,normal}", "y(4) global_group","y(5) <0.11.0>", "0xb7ce7764 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.11.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,84}, {trap_exit,true}]}, {<0.20.0>, [{registered_name,file_server_2}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6e0fea0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) file_server","y(3) #Port<0.223>", "y(4) file_server_2","y(5) <0.11.0>", "0xb6e0febc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,156}]}, {heap_size,6765}, {total_heap_size,24476}, {links,[#Port<0.223>,<0.11.0>]}, {memory,98324}, {message_queue_len,0}, {reductions,602674}, {trap_exit,true}]}, {<0.21.0>, [{registered_name,code_server}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7595254 (code_server:loop/1 + 64)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6db1450 Return addr 0x081f9e74 ()", "y(0) {state,<0.11.0>,\"/opt/membase/1.7.0r/erlang/lib/erlang\",[\"/opt/membase/1.7.0r/bin", "y(1) <0.11.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,49}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[<0.11.0>]}, {memory,131692}, {message_queue_len,0}, {reductions,207538}, {trap_exit,true}]}, {<0.22.0>, [{registered_name,standard_error_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf2790 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,standard_error,<0.23.0>,<0.23.0>,{local,standard_error_sup}}", "y(4) standard_error_sup","y(5) <0.11.0>", "0xb7cf27ac Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.11.0>,<0.23.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<0.23.0>, [{registered_name,standard_error}, {status,waiting}, {initial_call,{standard_error,server,2}}, {backtrace, ["Program counter: 0xb75925d4 (standard_error:server_loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1183c Return addr 0x081f9e74 ()", "y(0) #Port<0.506>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.22.0>,#Port<0.506>]}, {memory,1352}, {message_queue_len,0}, {reductions,7}, {trap_exit,true}]}, {<0.24.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d15bb0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,user_sup,<0.25.0>,<0.25.0>,{<0.24.0>,user_sup}}", "y(4) <0.24.0>","y(5) <0.11.0>", "0xb7d15bcc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<0.11.0>,<0.25.0>]}, {memory,6808}, {message_queue_len,0}, {reductions,71}, {trap_exit,true}]}, {<0.25.0>, [{registered_name,user}, {status,waiting}, {initial_call,{user,server,2}}, {backtrace, ["Program counter: 0xb75a60cc (user:server_loop/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d3aa20 Return addr 0x081f9e74 ()", "y(0) []","y(1) []","y(2) {[],[]}", "y(3) #Port<0.523>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<0.6.0>,<0.24.0>,#Port<0.523>]}, {memory,54560}, {message_queue_len,0}, {reductions,28052}, {trap_exit,true}]}, {<0.26.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce73a0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) kernel_config","y(3) []", "y(4) <0.26.0>","y(5) <0.11.0>", "0xb7ce73bc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.11.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,176}, {trap_exit,true}]}, {<0.27.0>, [{registered_name,kernel_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d65c30 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_safe_sup},one_for_one,[{child,<0.92.0>,dets,{dets_server,sta", "y(4) kernel_safe_sup","y(5) <0.11.0>", "0xb7d65c4c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<0.87.0>,<0.91.0>,<0.92.0>,<0.88.0>,<0.50.0>, <0.56.0>,<0.11.0>]}, {memory,3536}, {message_queue_len,0}, {reductions,438}, {trap_exit,true}]}, {<0.32.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1147c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.33.0>,{appl_data,sasl,[sasl_sup,alarm_handler,overload,release_handler]", "y(2) <0.7.0>", "0xb7d1148c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.7.0>,<0.33.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<0.33.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfa8f8 Return addr 0x081f9e74 ()", "y(0) {state,tty,undefined}","y(1) sasl", "y(2) <0.34.0>","y(3) <0.32.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.32.0>,<0.34.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,70}, {trap_exit,true}]}, {<0.34.0>, [{registered_name,sasl_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d108a8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_sup},one_for_one,[{child,<0.38.0>,release_handler,{release_han", "y(4) sasl_sup","y(5) <0.33.0>", "0xb7d108c4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.35.0>,<0.38.0>,<0.33.0>]}, {memory,2880}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<0.35.0>, [{registered_name,sasl_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10ff8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_safe_sup},one_for_one,[{child,<0.37.0>,overload,{overload,star", "y(4) sasl_safe_sup","y(5) <0.34.0>", "0xb7d11014 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.36.0>,<0.37.0>,<0.34.0>]}, {memory,2880}, {message_queue_len,0}, {reductions,174}, {trap_exit,true}]}, {<0.36.0>, [{registered_name,alarm_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10c54 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,alarm_handler,false,[{system_memory_high_watermark,[]}],false}]", "y(3) alarm_handler","y(4) <0.35.0>", "0xb7d10c6c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.35.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,43}, {trap_exit,true}]}, {<0.37.0>, [{registered_name,overload}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf2ee0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) overload", "y(3) {state,0,0,8.000000e-01,171,1.000000e-01,{0,0},clear}", "y(4) overload","y(5) <0.35.0>", "0xb7cf2efc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.35.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,39}, {trap_exit,false}]}, {<0.38.0>, [{registered_name,release_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1f758 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) release_handler", "y(3) {state,[],\"/opt/membase/1.7.0r/erlang/lib/erlang\",\"/opt/membase/1.7.0r/erlang/lib", "y(4) release_handler","y(5) <0.34.0>", "0xb7d1f774 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,4181}, {links,[<0.34.0>]}, {memory,17124}, {message_queue_len,0}, {reductions,2238}, {trap_exit,false}]}, {<0.41.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf23f4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.42.0>,{appl_data,os_mon,[os_mon_sup,os_mon_sysinfo,disksup,memsup,cpu_s", "y(2) <0.7.0>", "0xb7cf2404 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.7.0>,<0.42.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<0.42.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfaca0 Return addr 0x081f9e74 ()", "y(0) []","y(1) os_mon","y(2) <0.43.0>", "y(3) <0.41.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.41.0>,<0.43.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<0.43.0>, [{registered_name,os_mon_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d34058 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,os_mon_sup},one_for_one,[{child,<0.47.0>,cpu_sup,{cpu_sup,start_lin", "y(4) os_mon_sup","y(5) <0.42.0>", "0xb7d34074 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.44.0>,<0.45.0>,<0.47.0>,<0.42.0>]}, {memory,3476}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<0.44.0>, [{registered_name,disksup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6db8d64 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) disksup", "y(3) {state,80,60000,{unix,linux},[{\"/\",11173104,41},{\"/boot\",101086,13},{\"/dev/shm\",1", "y(4) disksup","y(5) <0.43.0>", "0xb6db8d80 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,32}]}, {heap_size,2584}, {total_heap_size,5168}, {links,[<0.43.0>,#Port<0.876>]}, {memory,21092}, {message_queue_len,0}, {reductions,27167}, {trap_exit,true}]}, {<0.45.0>, [{registered_name,memsup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb769597c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) memsup", "y(3) {state,{unix,linux},true,{1765871616,2075373568},{<0.414.0>,786412},false,60000,3", "y(4) memsup","y(5) <0.43.0>", "0xb7695998 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,36}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<0.43.0>,<0.46.0>]}, {memory,13196}, {message_queue_len,0}, {reductions,43658}, {trap_exit,true}]}, {<0.46.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7611e94 (memsup:port_idle/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1d504 Return addr 0x081f9e74 ()", "y(0) []","y(1) #Port<0.905>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,34}]}, {heap_size,2584}, {total_heap_size,2961}, {links,[<0.45.0>,#Port<0.905>]}, {memory,12204}, {message_queue_len,0}, {reductions,12223}, {trap_exit,true}]}, {<0.47.0>, [{registered_name,cpu_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d17850 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) cpu_sup", "y(3) {state,<0.48.0>,{unix,linux}}", "y(4) cpu_sup","y(5) <0.43.0>", "0xb7d1786c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.43.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,33}, {trap_exit,true}]}, {<0.48.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb760a4f4 (cpu_sup:measurement_server_loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf362c Return addr 0x081f9e74 ()", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []","y(5) []", "y(6) []","y(7) []", "y(8) {internal,<0.49.0>,[],{unix,linux}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.49.0>]}, {memory,1272}, {message_queue_len,0}, {reductions,11}, {trap_exit,true}]}, {<0.49.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb760ade0 (cpu_sup:port_server_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cea5b0 Return addr 0x081f9e74 ()", "y(0) []","y(1) 6000", "y(2) #Port<0.943>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.48.0>,#Port<0.943>]}, {memory,4308}, {message_queue_len,0}, {reductions,299}, {trap_exit,false}]}, {<0.50.0>, [{registered_name,timer_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf6ac4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) 449","y(2) timer", "y(3) []","y(4) timer_server", "y(5) <0.27.0>", "0xb7cf6ae0 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5077}]}, {heap_size,2584}, {total_heap_size,2961}, {links, [<0.458.0>,<0.478.0>,<0.1164.0>,<0.465.0>,<0.415.0>, <0.440.0>,<0.457.0>,<0.419.0>,<0.78.0>,<0.82.0>, <0.27.0>]}, {memory,12444}, {message_queue_len,0}, {reductions,1639491}, {trap_exit,true}]}, {<0.52.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d66f54 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.53.0>,{appl_data,ns_server,[ns_server_sup,ns_config,ns_config_sup,ns_co", "y(2) <0.7.0>", "0xb7d66f64 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.7.0>,<0.53.0>]}, {memory,2860}, {message_queue_len,0}, {reductions,48}, {trap_exit,true}]}, {<0.53.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cea958 Return addr 0x081f9e74 ()", "y(0) []","y(1) ns_server", "y(2) <0.54.0>","y(3) <0.52.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.52.0>,<0.54.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,50}, {trap_exit,true}]}, {<0.54.0>, [{registered_name,ns_server_cluster_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d31b9c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_cluster_sup},one_for_one,[{child,<0.414.0>,ns_server_sup,", "y(4) ns_server_cluster_sup","y(5) <0.53.0>", "0xb7d31bb8 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,2584}, {total_heap_size,2584}, {links, [<0.65.0>,<0.117.0>,<0.414.0>,<0.55.0>,<0.64.0>, <0.53.0>]}, {memory,10836}, {message_queue_len,0}, {reductions,1970}, {trap_exit,true}]}, {<0.55.0>, [{registered_name,dist_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10500 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) dist_manager", "y(3) {state,true,\"10.1.5.178\"}", "y(4) dist_manager","y(5) <0.54.0>", "0xb7d1051c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.54.0>]}, {memory,2840}, {message_queue_len,0}, {reductions,912}, {trap_exit,false}]}, {<0.56.0>, [{registered_name,inet_gethost_native_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf2b38 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,inet_gethost_native,<0.57.0>,<0.57.0>,{local,inet_gethost_native_sup}}", "y(4) inet_gethost_native_sup", "y(5) <0.27.0>", "0xb7cf2b54 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.27.0>,<0.57.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,41}, {trap_exit,true}]}, {<0.57.0>, [{registered_name,inet_gethost_native}, {status,waiting}, {initial_call,{inet_gethost_native,server_init,2}}, {backtrace, ["Program counter: 0xb7f44cc4 (inet_gethost_native:main_loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76caf84 Return addr 0x081f9e74 ()", "y(0) {state,#Port<0.984>,8000,8209,12306,<0.56.0>,4,{statistics,0,0,0,0,0,0,0,0}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,52}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.56.0>,#Port<0.984>]}, {memory,8316}, {message_queue_len,0}, {reductions,11359}, {trap_exit,true}]}, {<0.58.0>, [{registered_name,net_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce8900 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,net_sup},one_for_all,[{child,<0.61.0>,net_kernel,{net_kernel,start_", "y(4) net_sup","y(5) <0.11.0>", "0xb7ce891c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.59.0>,<0.60.0>,<0.61.0>,<0.11.0>]}, {memory,4408}, {message_queue_len,0}, {reductions,252}, {trap_exit,true}]}, {<0.59.0>, [{registered_name,erl_epmd}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce8ca8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) erl_epmd", "y(3) {state,#Port<0.1020>,21100,ns_1}", "y(4) erl_epmd","y(5) <0.58.0>", "0xb7ce8cc4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.58.0>,#Port<0.1020>]}, {memory,1352}, {message_queue_len,0}, {reductions,127}, {trap_exit,false}]}, {<0.60.0>, [{registered_name,auth}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d0d318 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) auth", "y(3) {state,lulxzmudqfmlvwvs,16403}", "y(4) auth","y(5) <0.58.0>", "0xb7d0d334 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.58.0>]}, {memory,2840}, {message_queue_len,0}, {reductions,379}, {trap_exit,true}]}, {<0.61.0>, [{registered_name,net_kernel}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf1314 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) net_kernel", "y(3) {state,'ns_1@10.1.5.178','ns_1@10.1.5.178',longnames,{tick,<0.63.0>,15000},7000,s", "y(4) net_kernel","y(5) <0.58.0>", "0xb7cf1330 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,610}, {total_heap_size,987}, {links, [<0.63.0>,<0.411.0>,<0.423.0>,<0.58.0>,<0.62.0>, #Port<0.1018>]}, {memory,4464}, {message_queue_len,0}, {reductions,1999}, {trap_exit,true}]}, {<0.62.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,accept_loop,2}}, {backtrace, ["Program counter: 0xb7e67f40 (prim_inet:accept0/2 + 92)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d588c8 Return addr 0xb762fcf4 (inet_tcp:accept/1 + 20)", "y(0) 84","y(1) #Port<0.1018>", "0xb7d588d4 Return addr 0xb7632ff0 (inet_tcp_dist:accept_loop/2 + 48)", "y(0) []", "0xb7d588dc Return addr 0x081f9e74 ()", "y(0) []","y(1) #Port<0.1018>", "y(2) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.61.0>]}, {memory,3396}, {message_queue_len,0}, {reductions,789}, {trap_exit,false}]}, {<0.63.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{net_kernel,ticker,2}}, {backtrace, ["Program counter: 0xb75857dc (net_kernel:ticker_loop/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce9650 Return addr 0x081f9e74 ()", "y(0) 15000","y(1) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.61.0>]}, {memory,1272}, {message_queue_len,0}, {reductions,151}, {trap_exit,false}]}, {<0.64.0>, [{registered_name,ns_cluster}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76d0178 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_cluster","y(3) {state}", "y(4) ns_cluster","y(5) <0.54.0>", "0xb76d0194 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,4181}, {total_heap_size,4181}, {links,[<0.54.0>]}, {memory,17124}, {message_queue_len,0}, {reductions,1605}, {trap_exit,false}]}, {<0.65.0>, [{registered_name,mb_mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d32530 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mb_mnesia_sup},one_for_one,[{child,<0.67.0>,mb_mnesia,{mb_mnesia,st", "y(4) mb_mnesia_sup","y(5) <0.54.0>", "0xb7d3254c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.66.0>,<0.67.0>,<0.54.0>]}, {memory,2880}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<0.66.0>, [{registered_name,mb_mnesia_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d44384 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []","y(2) []", "y(3) mb_mnesia_events","y(4) <0.65.0>", "0xb7d4439c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.65.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,32}, {trap_exit,true}]}, {<0.67.0>, [{registered_name,mb_mnesia}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76b9260 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mb_mnesia", "y(3) {state,['ns_1@10.1.5.178']}", "y(4) mb_mnesia","y(5) <0.65.0>", "0xb76b927c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<0.65.0>,<0.76.0>]}, {memory,10756}, {message_queue_len,0}, {reductions,4729}, {trap_exit,true}]}, {<0.70.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d45814 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.71.0>,{appl_data,mnesia,[mnesia_dumper_load_regulator,mnesia_event,mnes", "y(2) <0.7.0>", "0xb7d45824 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.7.0>,<0.71.0>]}, {memory,2860}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<0.71.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cead00 Return addr 0x081f9e74 ()", "y(0) {normal,[]}","y(1) mnesia_sup", "y(2) <0.72.0>","y(3) <0.70.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.70.0>,<0.72.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<0.72.0>, [{registered_name,mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d646d8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_sup},one_for_all,[{child,<0.74.0>,mnesia_kernel_sup,{mnesia_", "y(4) mnesia_sup","y(5) <0.71.0>", "0xb7d646f4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.73.0>,<0.74.0>,<0.71.0>]}, {memory,3456}, {message_queue_len,0}, {reductions,207}, {trap_exit,true}]}, {<0.73.0>, [{registered_name,mnesia_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cedd84 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,mnesia_event,false,{state,[],false,[]},false}]", "y(3) mnesia_event","y(4) <0.72.0>", "0xb7cedd9c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.72.0>,<0.76.0>]}, {memory,3436}, {message_queue_len,0}, {reductions,408}, {trap_exit,true}]}, {<0.74.0>, [{registered_name,mnesia_kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1311c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_kernel_sup},one_for_all,[{child,<0.83.0>,mnesia_late_loader,", "y(4) mnesia_kernel_sup","y(5) <0.72.0>", "0xb7d13138 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,610}, {total_heap_size,987}, {links, [<0.77.0>,<0.81.0>,<0.82.0>,<0.83.0>,<0.79.0>, <0.80.0>,<0.78.0>,<0.75.0>,<0.76.0>,<0.72.0>]}, {memory,4528}, {message_queue_len,0}, {reductions,599}, {trap_exit,true}]}, {<0.75.0>, [{registered_name,mnesia_monitor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfec84 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_monitor", "y(3) {state,<0.74.0>,[],[],true,[],undefined,[]}", "y(4) mnesia_monitor","y(5) <0.74.0>", "0xb7cfeca0 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,28}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<0.89.0>,<0.74.0>]}, {memory,13196}, {message_queue_len,0}, {reductions,6495}, {trap_exit,true}]}, {<0.76.0>, [{registered_name,mnesia_subscr}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d45460 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_subscr", "y(3) {state,<0.74.0>,20503}", "y(4) mnesia_subscr","y(5) <0.74.0>", "0xb7d4547c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.73.0>,<0.74.0>,<0.67.0>]}, {memory,1372}, {message_queue_len,0}, {reductions,111}, {trap_exit,true}]}, {<0.77.0>, [{registered_name,mnesia_locker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb73bd100 (mnesia_locker:loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7693100 Return addr 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []", "y(5) {state,<0.74.0>}", "0xb769311c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(1) mnesia_locker","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0xb7693134 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,356}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.74.0>]}, {memory,8296}, {message_queue_len,0}, {reductions,194367}, {trap_exit,true}]}, {<0.78.0>, [{registered_name,mnesia_recover}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d19390 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_recover", "y(3) {state,<0.74.0>,undefined,undefined,undefined,0,true,[]}", "y(4) mnesia_recover","y(5) <0.74.0>", "0xb7d193ac Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<0.74.0>,<0.50.0>]}, {memory,5876}, {message_queue_len,0}, {reductions,2859}, {trap_exit,true}]}, {<0.79.0>, [{registered_name,mnesia_tm}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb73e6d44 (mnesia_tm:doit_loop/1 + 108)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d4231c Return addr 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(0) []","y(1) []", "y(2) {state,{0,nil},{0,nil},<0.74.0>,[],[],[]}", "y(3) []","y(4) []","y(5) <0.74.0>", "y(6) {0,nil}","y(7) {0,nil}", "0xb7d42340 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(1) mnesia_tm","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0xb7d42358 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,106}]}, {heap_size,2584}, {total_heap_size,2961}, {links,[<0.74.0>]}, {memory,12244}, {message_queue_len,0}, {reductions,446853}, {trap_exit,true}]}, {<0.80.0>, [{registered_name,mnesia_checkpoint_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d593b0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_checkpoint_sup},simple_one_for_one,[{child,undefined,mnesia_", "y(4) mnesia_checkpoint_sup","y(5) <0.74.0>", "0xb7d593cc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.74.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<0.81.0>, [{registered_name,mnesia_snmp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf3288 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_snmp_sup},simple_one_for_one,[{child,undefined,mnesia_snmp_s", "y(4) mnesia_snmp_sup","y(5) <0.74.0>", "0xb7cf32a4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.74.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<0.82.0>, [{registered_name,mnesia_controller}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6fda380 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_controller", "y(3) {state,<0.74.0>,true,[],[],{0,nil},[],[],{0,nil},undefined,[],[],{interval,#Ref<0", "y(4) mnesia_controller","y(5) <0.74.0>", "0xb6fda39c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.74.0>,<0.50.0>]}, {memory,3436}, {message_queue_len,0}, {reductions,943}, {trap_exit,true}]}, {<0.83.0>, [{registered_name,mnesia_late_loader}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb738764c (mnesia_late_loader:loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cee704 Return addr 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(0) []","y(1) []", "y(2) {state,<0.74.0>}", "0xb7cee714 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(1) mnesia_late_loader","y(2) []", "y(3) []","y(4) [<0.74.0>]", "0xb7cee72c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.74.0>]}, {memory,2840}, {message_queue_len,0}, {reductions,174}, {trap_exit,false}]}, {<0.87.0>, [{registered_name,disk_log_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d5fd80 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,disk_log_sup},simple_one_for_one,[{child,undefined,disk_log,{disk_l", "y(4) disk_log_sup","y(5) <0.27.0>", "0xb7d5fd9c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,19}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<0.89.0>,<0.27.0>]}, {memory,6808}, {message_queue_len,0}, {reductions,3694}, {trap_exit,true}]}, {<0.88.0>, [{registered_name,disk_log_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb769ac98 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) disk_log_server","y(3) {state,[]}", "y(4) disk_log_server","y(5) <0.27.0>", "0xb769acb4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,45}]}, {heap_size,987}, {total_heap_size,1974}, {links,[<0.89.0>,<0.27.0>]}, {memory,8316}, {message_queue_len,0}, {reductions,3975}, {trap_exit,true}]}, {<0.89.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb741f800 (disk_log:loop/1 + 84)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6db5f30 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) {state,[],[],<0.87.0>,<0.88.0>,100,{arg,latest_log,undefined,\"/var/membase/mnesia", "0xb6db5f38 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,1220}, {links,[<0.87.0>,<0.88.0>,<0.75.0>,#Port<0.2892>]}, {memory,5340}, {message_queue_len,0}, {reductions,140003}, {trap_exit,true}]}, {<0.91.0>, [{registered_name,dets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76be338 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,dets_sup},simple_one_for_one,[{child,undefined,dets,{dets,istart_li", "y(4) dets_sup","y(5) <0.27.0>", "0xb76be354 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.27.0>]}, {memory,4348}, {message_queue_len,0}, {reductions,1139}, {trap_exit,true}]}, {<0.92.0>, [{registered_name,dets}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76cbed8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) dets_server", "y(3) {state,40996,[<0.27.0>],[]}", "y(4) dets","y(5) <0.27.0>", "0xb76cbef4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<0.27.0>]}, {memory,5856}, {message_queue_len,0}, {reductions,1613}, {trap_exit,true}]}, {<0.117.0>, [{registered_name,ns_config_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d52c28 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_config_sup},rest_for_one,[{child,undefined,ns_config_log,{ns_con", "y(4) ns_config_sup","y(5) <0.54.0>", "0xb7d52c44 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<0.118.0>,<0.119.0>,<0.54.0>]}, {memory,27500}, {message_queue_len,0}, {reductions,580}, {trap_exit,true}]}, {<0.118.0>, [{registered_name,ns_config_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x081f9e70 (unknown function)", "CP: 0x081f9e74 ()", "arity = 3"," proc_lib"," wake_up", " [gen_event,wake_hib,[<0.117.0>,ns_config_events,[{handler,ns_pubsub,#Ref<0.0.0.6817>,{s"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,779}, {total_heap_size,779}, {links,[<0.440.0>,<0.475.0>,<0.481.0>,<0.117.0>]}, {memory,3656}, {message_queue_len,0}, {reductions,251580}, {trap_exit,true}]}, {<0.119.0>, [{registered_name,ns_config}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6de7acc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_config", "y(3) {config,{full,\"/etc/membase/priv/config\",undefined,ns_config_default},[[{director", "y(4) ns_config","y(5) <0.117.0>", "0xb6de7ae8 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,46368}, {total_heap_size,167761}, {links,[<0.117.0>]}, {memory,671444}, {message_queue_len,0}, {reductions,84935}, {trap_exit,true}]}, {<0.411.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_accept,6}}, {backtrace, ["Program counter: 0xb726b6e4 (dist_util:con_loop/9 + 72)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6f88f08 Return addr 0x081f9e74 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,9040,4348,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.178'", "y(6) {net_address,{{10,1,5,192},37359},\"10.1.5.192\",tcp,inet}", "y(7) #Port<0.2378>", "y(8) 'ns_1@10.1.5.192'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.61.0>,#Port<0.2378>]}, {memory,8256}, {message_queue_len,0}, {reductions,1772}, {trap_exit,false}]}, {<0.414.0>, [{registered_name,ns_server_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb70e3880 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_sup},one_for_one,[{child,<0.483.0>,moxi_stats_collector,{", "y(4) ns_server_sup","y(5) <0.54.0>", "0xb70e389c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,75025}, {total_heap_size,196418}, {links, [<0.460.0>,<0.474.0>,<0.480.0>,<0.481.0>,<0.483.0>, <0.477.0>,<0.478.0>,<0.475.0>,<0.472.0>,<0.473.0>, <0.466.0>,<0.417.0>,<0.457.0>,<0.458.0>,<0.440.0>, <0.415.0>,<0.416.0>,<0.54.0>]}, {memory,786412}, {message_queue_len,0}, {reductions,85274}, {trap_exit,true}]}, {<0.415.0>, [{registered_name,ns_log}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76d42d0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) ns_log", "y(3) {state,[{log_entry,{1302,303540,831821},'ns_1@10.1.5.178',ns_node_disco,3,\"Initia", "y(4) ns_log","y(5) <0.414.0>", "0xb76d42ec Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,4181}, {total_heap_size,10946}, {links,[<0.50.0>,<0.414.0>]}, {memory,44204}, {message_queue_len,0}, {reductions,3323}, {trap_exit,false}]}, {<0.416.0>, [{registered_name,ns_log_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d04cbc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []","y(2) []", "y(3) ns_log_events","y(4) <0.414.0>", "0xb7d04cd4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.414.0>]}, {memory,4348}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<0.417.0>, [{registered_name,ns_node_disco_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d2ce78 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_node_disco_sup},rest_for_one,[{child,<0.430.0>,ns_config_rep,{ns", "y(4) ns_node_disco_sup","y(5) <0.414.0>", "0xb7d2ce94 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.419.0>,<0.430.0>,<0.418.0>,<0.414.0>]}, {memory,2900}, {message_queue_len,0}, {reductions,793}, {trap_exit,true}]}, {<0.418.0>, [{registered_name,ns_node_disco_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d2eb1c Return addr 0xb7e88088 (proc_lib:wake_up/3 + 60)", "y(0) false","y(1) []", "y(2) [{handler,menelaus_event,ns_node_disco_events,{state,ns_node_disco_events,undefin", "y(3) ns_node_disco_events","y(4) <0.417.0>", "0xb7d2eb34 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88098 (proc_lib:wake_up/3 + 76)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,322}, {links,[<0.417.0>]}, {memory,1768}, {message_queue_len,0}, {reductions,73}, {trap_exit,true}]}, {<0.419.0>, [{registered_name,ns_node_disco}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7164c70 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_node_disco", "y(3) {state,false,{interval,#Ref<0.0.0.6490>}}", "y(4) ns_node_disco","y(5) <0.417.0>", "0xb7164c8c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,45}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<0.417.0>,<0.50.0>]}, {memory,300520}, {message_queue_len,0}, {reductions,3737}, {trap_exit,false}]}, {<0.423.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_accept,6}}, {backtrace, ["Program counter: 0xb726b6e4 (dist_util:con_loop/9 + 72)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d2e768 Return addr 0x081f9e74 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,2001,2016,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.178'", "y(6) {net_address,{{10,1,5,193},38404},\"10.1.5.193\",tcp,inet}", "y(7) #Port<0.2383>", "y(8) 'ns_1@10.1.5.193'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.61.0>,#Port<0.2383>]}, {memory,8256}, {message_queue_len,0}, {reductions,1772}, {trap_exit,false}]}, {<0.430.0>, [{registered_name,ns_config_rep}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6e423dc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_config_rep","y(3) {state}", "y(4) ns_config_rep","y(5) <0.417.0>", "0xb6e423f8 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,46368}, {total_heap_size,92736}, {links,[<0.417.0>]}, {memory,371344}, {message_queue_len,0}, {reductions,15836}, {trap_exit,false}]}, {<0.440.0>, [{registered_name,mb_master}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb71d63b4 (gen_fsm:loop/7 + 156)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6ded51c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mb_master", "y(3) {state,undefined,'ns_1@10.1.5.192',['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.", "y(4) candidate","y(5) mb_master", "y(6) <0.414.0>", "0xb6ded53c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,49}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.118.0>,<0.414.0>,<0.50.0>]}, {memory,8336}, {message_queue_len,0}, {reductions,40574}, {trap_exit,false}]}, {<0.457.0>, [{registered_name,ns_heart}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76c4cf4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_heart", "y(3) [{meminfo,<<771 bytes>>},{replication,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9c", "y(4) ns_heart","y(5) <0.414.0>", "0xb76c4d10 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<0.414.0>,<0.50.0>]}, {memory,27480}, {message_queue_len,0}, {reductions,1416520}, {trap_exit,false}]}, {<0.458.0>, [{registered_name,ns_doctor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76799a8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_doctor", "y(3) {state,{dict,3,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{", "y(4) ns_doctor","y(5) <0.414.0>", "0xb76799c4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<0.414.0>,<0.50.0>]}, {memory,71264}, {message_queue_len,0}, {reductions,56906}, {trap_exit,false}]}, {<0.460.0>, [{registered_name,menelaus_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6debc28 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,menelaus_sup},one_for_one,[{child,<0.465.0>,menelaus_web_alerts_srv", "y(4) menelaus_sup","y(5) <0.414.0>", "0xb6debc44 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[<0.461.0>,<0.465.0>,<0.1164.0>,<0.414.0>]}, {memory,131812}, {message_queue_len,0}, {reductions,10925}, {trap_exit,true}]}, {<0.461.0>, [{registered_name,menelaus_web}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf9f4c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mochiweb_socket_server", "y(3) {mochiweb_socket_server,8091,#Fun,{local,menelaus_web}", "y(4) menelaus_web","y(5) <0.460.0>", "0xb7cf9f68 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,987}, {links, [<0.479.0>,<0.485.0>,<0.1165.0>,<0.460.0>,<0.462.0>, #Port<0.2388>]}, {memory,4448}, {message_queue_len,0}, {reductions,365}, {trap_exit,true}]}, {<0.462.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb720b578 (menelaus_web:handle_streaming/4 + 196)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6fa4f8c Return addr 0xb7209848 (menelaus_web:loop/3 + 13188)", "y(0) {struct,[{buckets,[]}]}", "y(1) {struct,[{buckets,[]}]}", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBuck", "y(3) {mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(4) #Fun", "0xb6fa4fa4 Return addr 0xb71f5880 (mochiweb_http:headers/5 + 680)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(5) Catch 0xb7209858 (menelaus_web:loop/3 + 13204)", "0xb6fa4fc0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "0xb6fa4fd4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,28657}, {total_heap_size,28657}, {links,[<0.461.0>,#Port<0.2393>]}, {memory,115128}, {message_queue_len,0}, {reductions,5214}, {trap_exit,false}]}, {<0.465.0>, [{registered_name,menelaus_web_alerts_srv}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb71422f0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) menelaus_web_alerts_srv", "y(3) {state,[],[],{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],", "y(4) menelaus_web_alerts_srv", "y(5) <0.460.0>", "0xb714230c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,323}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<0.50.0>,<0.460.0>]}, {memory,33868}, {message_queue_len,0}, {reductions,188377}, {trap_exit,false}]}, {<0.466.0>, [{registered_name,ns_port_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6ff66d8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_port_sup},one_for_one,[{child,<0.488.0>,{memcached,\"./bin/memcac", "y(4) ns_port_sup","y(5) <0.414.0>", "0xb6ff66f4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<0.468.0>,<0.488.0>,<0.414.0>]}, {memory,300540}, {message_queue_len,0}, {reductions,3234}, {trap_exit,true}]}, {<0.468.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d5ee10 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi,5000,{1302,303852,128447},<0.469.0>}", "y(4) <0.468.0>","y(5) <0.466.0>", "0xb7d5ee2c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<0.466.0>,<0.469.0>]}, {memory,10756}, {message_queue_len,0}, {reductions,150}, {trap_exit,true}]}, {<0.469.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf95c0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2389>,moxi,{[\"2011-04-08 16:04:12: (cproxy_config.c.325) env: MOXI", "y(4) <0.469.0>","y(5) <0.468.0>", "0xb7cf95dc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<0.468.0>,#Port<0.2389>]}, {memory,13196}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<0.472.0>, [{registered_name,ns_tick_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d05c30 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.6760>,{state,#Fun,ignored},<", "y(3) ns_tick_event","y(4) <0.414.0>", "0xb7d05c48 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,61}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.477.0>,<0.414.0>]}, {memory,4368}, {message_queue_len,0}, {reductions,10252}, {trap_exit,true}]}, {<0.473.0>, [{registered_name,ns_stats_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d0979c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.6806>,{state,#Fun,ignored},<", "y(3) ns_stats_event","y(4) <0.414.0>", "0xb7d097b4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,421}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<0.478.0>,<0.414.0>]}, {memory,13196}, {message_queue_len,0}, {reductions,28911}, {trap_exit,true}]}, {<0.474.0>, [{registered_name,ns_bucket_worker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d43c4c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) work_queue","y(3) []", "y(4) ns_bucket_worker","y(5) <0.414.0>", "0xb7d43c68 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,1597}, {links,[<0.414.0>]}, {memory,6788}, {message_queue_len,0}, {reductions,288}, {trap_exit,false}]}, {<0.475.0>, [{registered_name,ns_bucket_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb70f4d48 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_bucket_sup},one_for_one,[{child,<0.765.0>,{per_bucket_sup,\"Failo", "y(4) ns_bucket_sup","y(5) <0.414.0>", "0xb70f4d64 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<0.414.0>,<0.765.0>,<0.118.0>]}, {memory,71284}, {message_queue_len,0}, {reductions,246}, {trap_exit,true}]}, {<0.477.0>, [{registered_name,system_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb77703b0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) system_stats_collector", "y(3) {state,#Port<0.2392>,[{cpu_local_ms,2425406020},{cpu_idle_ms,2395487550},{swap_to", "y(4) system_stats_collector", "y(5) <0.414.0>", "0xb77703cc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,166}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.414.0>,<0.472.0>,#Port<0.2392>]}, {memory,8336}, {message_queue_len,0}, {reductions,46264}, {trap_exit,false}]}, {<0.478.0>, [{registered_name,'stats_archiver-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6dad2e0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"@system\"}", "y(4) 'stats_archiver-@system'", "y(5) <0.414.0>", "0xb6dad2fc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1899}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<0.414.0>,<0.473.0>,<0.50.0>]}, {memory,33888}, {message_queue_len,0}, {reductions,4245733}, {trap_exit,false}]}, {<0.479.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb720b578 (menelaus_web:handle_streaming/4 + 196)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76e574c Return addr 0xb7240ed0 (menelaus_web_buckets:checking_bucket_access/4 + 260)", "y(0) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(1) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsS", "y(3) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) #Fun", "0xb76e5764 Return addr 0xb72097a0 (menelaus_web:loop/3 + 13020)", "y(0) #Ref<0.0.0.6889>", "y(1) Catch 0xb7240f00 (menelaus_web_buckets:checking_bucket_access/4 + 308)", "y(2) []", "y(3) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) []", "0xb76e577c Return addr 0xb71f5880 (mochiweb_http:headers/5 + 680)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(5) Catch 0xb7209858 (menelaus_web:loop/3 + 13204)", "0xb76e5798 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "0xb76e57ac Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,17711}, {total_heap_size,92736}, {links,[<0.461.0>,#Port<0.2396>]}, {memory,371444}, {message_queue_len,0}, {reductions,1243207}, {trap_exit,false}]}, {<0.480.0>, [{registered_name,'stats_reader-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfb620 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"@system\"}", "y(4) 'stats_reader-@system'", "y(5) <0.414.0>", "0xb7cfb63c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.414.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,26}, {trap_exit,false}]}, {<0.481.0>, [{registered_name,ns_moxi_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7106208 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_moxi_sup},one_for_one,[{child,<0.482.0>,{\"FailoverNodesTests-4a4", "y(4) ns_moxi_sup","y(5) <0.414.0>", "0xb7106224 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<0.414.0>,<0.482.0>,<0.118.0>]}, {memory,71284}, {message_queue_len,0}, {reductions,2236}, {trap_exit,true}]}, {<0.482.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d046d0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2394>,moxi,{[\"2011-04-08 16:04:12: (cproxy_config.c.316) env: MOXI", "y(4) <0.482.0>","y(5) <0.481.0>", "0xb7d046ec Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<0.481.0>,#Port<0.2394>]}, {memory,6808}, {message_queue_len,0}, {reductions,168}, {trap_exit,true}]}, {<0.483.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce99e0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi_stats_collector,5000,{1302,303852,147786},<0.484.0>}", "y(4) <0.483.0>","y(5) <0.414.0>", "0xb7ce99fc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.414.0>,<0.484.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<0.484.0>, [{registered_name,moxi_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb775fe30 Return addr 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(0) []","y(1) infinity", "y(2) moxi_stats_collector", "y(3) {state,#Port<0.2395>,{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[", "y(4) <0.484.0>","y(5) <0.483.0>", "0xb775fe4c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(1) moxi_stats_collector","y(2) []", "y(3) moxi_stats_collector", "y(4) {local,moxi_stats_collector}", "y(5) <0.483.0>","y(6) <0.483.0>", "0xb775fe6c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,32}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[#Port<0.2438>,<0.483.0>,#Port<0.2395>]}, {memory,131792}, {message_queue_len,0}, {reductions,2259506}, {trap_exit,false}]}, {<0.485.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ec02d4 (gen_server:do_multi_call/4 + 384)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d3eb0c Return addr 0xb7f0f760 (rpc:do_multicall/5 + 160)", "y(0) []","y(1) #Ref<0.0.0.23941>", "y(2) <0.1166.0>","y(3) #Ref<0.0.0.23940>", "0xb7d3eb20 Return addr 0xb71c14dc (diag_handler:diag_multicall/3 + 116)", "y(0) []","y(1) []", "0xb7d3eb2c Return addr 0xb71c1aec (diag_handler:handle_diag/1 + 192)", "y(0) ['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193']", "y(1) []","y(2) []","y(3) []", "0xb7d3eb40 Return addr 0xb720959c (menelaus_web:loop/3 + 12504)", "y(0) []", "y(1) [[\"2011\",45,[\"0\",\"4\"],45,[\"0\",\"8\"],32,\"15\",58,\"59\",58,[\"0\",\"0\"],46,\"831\"],32,\"ns_", "y(2) [{\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",[{num_replicas,1},{ra", "y(3) {mochiweb_request,#Port<0.2929>,'GET',\"/diag\",{1,0},{5,{\"user-agent\",{'User-Agent", "0xb7d3eb54 Return addr 0xb71f5880 (mochiweb_http:headers/5 + 680)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2929>,'GET',\"/diag\",{1,0},{5,{\"user-agent\",{'User-Agent", "y(5) Catch 0xb7209858 (menelaus_web:loop/3 + 13204)", "0xb7d3eb70 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2929>,'GET',\"/diag\",{1,0},{5,{\"user-agent\",{'User-Agent", "0xb7d3eb84 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[<0.461.0>,#Port<0.2929>]}, {memory,131852}, {message_queue_len,0}, {reductions,25262}, {trap_exit,false}]}, {<0.488.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d06ba0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,memcached,5000,{1302,303859,675513},<0.489.0>}", "y(4) <0.488.0>","y(5) <0.466.0>", "0xb7d06bbc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<0.466.0>,<0.489.0>]}, {memory,4368}, {message_queue_len,0}, {reductions,55}, {trap_exit,true}]}, {<0.489.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7698db8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2401>,memcached,{[empty],[empty,empty]},undefined,[],0}", "y(4) <0.489.0>","y(5) <0.488.0>", "0xb7698dd4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<0.488.0>,#Port<0.2401>]}, {memory,4368}, {message_queue_len,0}, {reductions,58}, {trap_exit,true}]}, {<0.765.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7e8844c (proc_lib:sync_wait/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d09b4c Return addr 0xb726f6b8 (single_bucket_sup:'-start_link/1-fun-0-'/2 + 172)", "y(0) infinity","y(1) <0.766.0>", "0xb7d09b58 Return addr 0x081f9e74 ()", "y(0) <0.475.0>", "y(1) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\""]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.475.0>,<0.766.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,30}, {trap_exit,true}]}, {<0.766.0>, [{registered_name, 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8844c (proc_lib:sync_wait/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d65268 Return addr 0xb7f0395c (supervisor:do_start_child/2 + 156)", "y(0) infinity","y(1) <0.767.0>", "0xb7d65274 Return addr 0xb7f0367c (supervisor:start_children/3 + 76)", "y(0) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(2) Catch 0xb7f0395c (supervisor:do_start_child/2 + 156)", "0xb7d65284 Return addr 0xb7f03370 (supervisor:init_children/2 + 136)", "y(0) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(1) []", "y(2) [{child,undefined,{ns_vbm_sup,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b9582", "y(3) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "0xb7d65298 Return addr 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(0) {state,{local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "0xb7d652a4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(1) 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(2) []","y(3) supervisor", "y(4) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(5) <0.765.0>","y(6) <0.765.0>", "0xb7d652c4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.765.0>,<0.767.0>]}, {memory,4368}, {message_queue_len,0}, {reductions,664}, {trap_exit,true}]}, {<0.767.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb761487c (timer:sleep/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb768ef90 Return addr 0xb71ecc34 (ns_memcached:ensure_bucket/2 + 456)", "y(0) 1000", "0xb768ef98 Return addr 0xb71eabc0 (ns_memcached:init/1 + 72)", "y(0) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "y(1) #Port<0.2571>","y(2) []", "0xb768efa8 Return addr 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(0) []","y(1) {1302,304045,481732}", "y(2) #Port<0.2571>", "y(3) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "0xb768efbc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(1) <0.767.0>","y(2) []", "y(3) ns_memcached","y(4) <0.767.0>", "y(5) <0.766.0>","y(6) <0.766.0>", "0xb768efdc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[<0.766.0>,#Port<0.2571>]}, {memory,131772}, {message_queue_len,0}, {reductions,1442625}, {trap_exit,false}]}, {<0.1164.0>, [{registered_name,hot_keys_keeper}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d44728 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) hot_keys_keeper", "y(3) {state,[],undefined,undefined}", "y(4) hot_keys_keeper","y(5) <0.460.0>", "0xb7d44744 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.460.0>,<0.50.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,56}, {trap_exit,false}]}, {<0.1165.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e67f40 (prim_inet:accept0/2 + 92)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d33260 Return addr 0xb762fcf4 (inet_tcp:accept/1 + 20)", "y(0) 46320","y(1) #Port<0.2388>", "0xb7d3326c Return addr 0xb71fcb04 (mochiweb_socket_server:acceptor_loop/1 + 80)", "y(0) []", "0xb7d33274 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) #Fun", "y(1) <0.461.0>", "y(2) Catch 0xb71fcb04 (mochiweb_socket_server:acceptor_loop/1 + 80)", "0xb7d33284 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.461.0>]}, {memory,1372}, {message_queue_len,0}, {reductions,38}, {trap_exit,false}]}, {<0.1166.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7ec05e4 (gen_server:rec_nodes/7 + 120)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d5c58c Return addr 0xb7ec3680 (gen_server:'-do_multi_call/4-fun-0-'/6 + 356)", "y(0) #Ref<0.0.0.23946>","y(1) 2000", "y(2) []","y(3) []","y(4) rex", "y(5) #Ref<0.0.0.23940>", "y(6) [{'ns_1@10.1.5.192',#Ref<0.0.0.23944>},{'ns_1@10.1.5.178',#Ref<0.0.0.23943>}]", "y(7) #Ref<0.0.0.23945>", "y(8) 'ns_1@10.1.5.193'", "0xb7d5c5b4 Return addr 0x081f9e74 ()", "y(0) #Ref<0.0.0.23940>","y(1) []", "y(2) []","y(3) []","y(4) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[]}, {memory,1452}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<0.1167.0>, [{registered_name,[]}, {status,running}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7df922c (unknown function)", "CP: 0xb71c1234 (diag_handler:grab_process_info/1 + 32)", "0xb6555020 Return addr 0xb71c2508 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 44)", "y(0) []", "0xb6555028 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) []","y(1) <0.1167.0>", "0xb6555034 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.1166.0>", "0xb6555040 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.1165.0>", "0xb655504c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,hot_keys_keeper},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.1164.0>", "0xb6555058 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.767.0>", "0xb6555064 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-5", "y(1) <0.766.0>", "0xb6555070 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.765.0>", "0xb655507c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.489.0>", "0xb6555088 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.488.0>", "0xb6555094 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.485.0>", "0xb65550a0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,moxi_stats_collector},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.484.0>", "0xb65550ac Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.483.0>", "0xb65550b8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.482.0>", "0xb65550c4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_moxi_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.481.0>", "0xb65550d0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,'stats_reader-@system'},{status,waiting},{initial_call,{proc_li", "y(1) <0.480.0>", "0xb65550dc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.479.0>", "0xb65550e8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,'stats_archiver-@system'},{status,waiting},{initial_call,{proc_", "y(1) <0.478.0>", "0xb65550f4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,system_stats_collector},{status,waiting},{initial_call,{proc_li", "y(1) <0.477.0>", "0xb6555100 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_bucket_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.475.0>", "0xb655510c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_bucket_worker},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.474.0>", "0xb6555118 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_stats_event},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.473.0>", "0xb6555124 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_tick_event},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.472.0>", "0xb6555130 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.469.0>", "0xb655513c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.468.0>", "0xb6555148 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_port_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.466.0>", "0xb6555154 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,menelaus_web_alerts_srv},{status,waiting},{initial_call,{proc_l", "y(1) <0.465.0>", "0xb6555160 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.462.0>", "0xb655516c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,menelaus_web},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.461.0>", "0xb6555178 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,menelaus_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.460.0>", "0xb6555184 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_doctor},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.458.0>", "0xb6555190 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_heart},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.457.0>", "0xb655519c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_master},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.440.0>", "0xb65551a8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config_rep},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.430.0>", "0xb65551b4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_accept,6}}", "y(1) <0.423.0>", "0xb65551c0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_node_disco},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.419.0>", "0xb65551cc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_node_disco_events},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.418.0>", "0xb65551d8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_node_disco_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.417.0>", "0xb65551e4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_log_events},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.416.0>", "0xb65551f0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_log},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.415.0>", "0xb65551fc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_server_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.414.0>", "0xb6555208 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_accept,6}}", "y(1) <0.411.0>", "0xb6555214 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.119.0>", "0xb6555220 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.118.0>", "0xb655522c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.117.0>", "0xb6555238 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,dets},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.92.0>", "0xb6555244 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,dets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.91.0>", "0xb6555250 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.89.0>", "0xb655525c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,disk_log_server},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.88.0>", "0xb6555268 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,disk_log_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.87.0>", "0xb6555274 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_late_loader},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.83.0>", "0xb6555280 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_controller},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.82.0>", "0xb655528c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_snmp_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.81.0>", "0xb6555298 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_checkpoint_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.80.0>", "0xb65552a4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_tm},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.79.0>", "0xb65552b0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_recover},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.78.0>", "0xb65552bc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_locker},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.77.0>", "0xb65552c8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_subscr},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.76.0>", "0xb65552d4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_monitor},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.75.0>", "0xb65552e0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_kernel_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.74.0>", "0xb65552ec Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_event},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.73.0>", "0xb65552f8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.72.0>", "0xb6555304 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.71.0>", "0xb6555310 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.70.0>", "0xb655531c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_mnesia},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.67.0>", "0xb6555328 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_mnesia_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.66.0>", "0xb6555334 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.65.0>", "0xb6555340 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_cluster},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.64.0>", "0xb655534c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{net_kernel,ticker,2}},{back", "y(1) <0.63.0>", "0xb6555358 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,accept_loop,2", "y(1) <0.62.0>", "0xb6555364 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,net_kernel},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.61.0>", "0xb6555370 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,auth},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.60.0>", "0xb655537c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,erl_epmd},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.59.0>", "0xb6555388 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,net_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.58.0>", "0xb6555394 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,inet_gethost_native},{status,waiting},{initial_call,{inet_getho", "y(1) <0.57.0>", "0xb65553a0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,inet_gethost_native_sup},{status,waiting},{initial_call,{proc_l", "y(1) <0.56.0>", "0xb65553ac Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,dist_manager},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.55.0>", "0xb65553b8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_server_cluster_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.54.0>", "0xb65553c4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.53.0>", "0xb65553d0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.52.0>", "0xb65553dc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,timer_server},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.50.0>", "0xb65553e8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.49.0>", "0xb65553f4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.48.0>", "0xb6555400 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,cpu_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.47.0>", "0xb655540c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.46.0>", "0xb6555418 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,memsup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.45.0>", "0xb6555424 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,disksup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.44.0>", "0xb6555430 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,os_mon_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.43.0>", "0xb655543c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.42.0>", "0xb6555448 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.41.0>", "0xb6555454 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,release_handler},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.38.0>", "0xb6555460 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,overload},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.37.0>", "0xb655546c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,alarm_handler},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.36.0>", "0xb6555478 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,sasl_safe_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.35.0>", "0xb6555484 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,sasl_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.34.0>", "0xb6555490 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.33.0>", "0xb655549c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.32.0>", "0xb65554a8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,kernel_safe_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.27.0>", "0xb65554b4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.26.0>", "0xb65554c0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,user},{status,waiting},{initial_call,{user,server,2}},{backtrac", "y(1) <0.25.0>", "0xb65554cc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.24.0>", "0xb65554d8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,standard_error},{status,waiting},{initial_call,{standard_error,", "y(1) <0.23.0>", "0xb65554e4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,standard_error_sup},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.22.0>", "0xb65554f0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,code_server},{status,waiting},{initial_call,{erlang,apply,2}},{", "y(1) <0.21.0>", "0xb65554fc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,file_server_2},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.20.0>", "0xb6555508 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,global_group},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.19.0>", "0xb6555514 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,inet_db},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.17.0>", "0xb6555520 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.16.0>", "0xb655552c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.15.0>", "0xb6555538 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.14.0>", "0xb6555544 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,global_name_server},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.13.0>", "0xb6555550 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,rex},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backt", "y(1) <0.12.0>", "0xb655555c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,kernel_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.11.0>", "0xb6555568 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.10.0>", "0xb6555574 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.9.0>", "0xb6555580 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,application_controller},{status,waiting},{initial_call,{erlang,", "y(1) <0.7.0>", "0xb655558c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,error_logger},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.6.0>", "0xb6555598 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,erl_prim_loader},{status,waiting},{initial_call,{erlang,apply,2", "y(1) <0.3.0>", "0xb65555a4 Return addr 0xb71c133c (diag_handler:do_diag_per_node/0 + 112)", "y(0) [{registered_name,init},{status,waiting},{initial_call,{otp_ring0,start,2}},{back", "y(1) <0.0.0>", "0xb65555b0 Return addr 0xb7f104f4 (rpc:'-handle_call_call/6-fun-0-'/5 + 104)", "y(0) []","y(1) []", "y(2) [{version,[{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"}", "y(3) [{alerts,[{email,[]},{email_alerts,false},{email_server,[{user,undefined},{pass,'", "y(4) [\"bucket_engine_1.6.5_10_g3c82cbf-Linux.i686.tar.gz\",\"curl-7.21.1-w64_patched.tar", "y(5) [{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"},{ns_serve", "0xb65555cc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7f104f4 (rpc:'-handle_call_call/6-fun-0-'/5 + 104)", "y(1) []","y(2) []","y(3) []", "y(4) <0.12.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,317811}, {total_heap_size,832040}, {links,[]}, {memory,3328520}, {message_queue_len,0}, {reductions,647692}, {trap_exit,false}]}]}, {memory,{2075373568,1765871616,{<0.414.0>,786412}}}, {disk, [{"/",11173104,41}, {"/boot",101086,13}, {"/dev/shm",1013364,0}]}]}, {'ns_1@10.1.5.192', [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {manifest, ["bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz", "curl-7.21.1-w64_patched.tar.gz", "ep-engine_1.6.5.3_143_g366c22e-Linux.x86_64.tar.gz", "google-perftools-1.7","google-perftools-1.7.tar.gz", "libconflate_1.6.5_1_g8a42194-Linux.x86_64.tar.gz", "libevent-2.0.7-rc.tar.gz", "libmemcached-0.41_trond-norbye_mingw32-revno895.tar.gz", "libvbucket_1.6.4_1_g5cb588e-Linux.x86_64.tar.gz", "membase-cli_1.6.5_2_gb9c8a7d-Linux.x86_64.tar.gz", "memcached_1.4.4_430_g6235145-Linux.x86_64.tar.gz", "moxi_1.6.5.1_16_g8128746-Linux.x86_64.tar.gz", "ns_server_1.6.5.3_291_gba5fe27.tar.gz", "portsigar_-Linux.x86_64.tar.gz", "sigar_sigar_1.6.4_402_g7809620-Linux.x86_64.tar.gz", "vbucketmigrator_1.7.0r_4_g8428c29-Linux.x86_64.tar.gz", "wallace_1.7.0r-100-g85ee8f5-Linux.x86_64-201104060842"]}, {config, [{alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,'filtered-out'}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up, server_joined,server_left,bucket_created, bucket_deleted,bucket_auth_failed]}]}, {buckets, [{'_vclock',[{'ns_1@10.1.5.192',{11,63469523235}}]}, {configs, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers, ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178']}, {map, [['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}]}]}, {memory_quota,1570}, {nodes_wanted, [{'_vclock',[{'ns_1@10.1.5.192',{2,63469523039}}]}, 'ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193']}, {otp, [{'_vclock',[{'ns_1@10.1.5.193',{1,63469523027}}]}, {cookie,lulxzmudqfmlvwvs}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {rebalance_status, {none, <<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>}}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds, [{'_vclock',[{'ns_1@10.1.5.192',{1,63469522657}}]}, {creds, [{"Administrator",[{password,'filtered-out'}]}]}]}, {{node,'ns_1@10.1.5.178',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.178',membership},inactiveFailed}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.178',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.192',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.192',membership},active}, {{node,'ns_1@10.1.5.192',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.192',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.192',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.192',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.193',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.193',membership},inactiveFailed}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.193',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}]}, {basic_info, [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1198}, {memory_data, {2058928128,1594331136,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}, {processes, [{<10463.0.0>, [{registered_name,init}, {status,waiting}, {initial_call,{otp_ring0,start,2}}, {backtrace, ["Program counter: 0x00002b2847ee2290 (init:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdb8000 Return addr 0x0000000000816eb8 ()", "y(0) {state,[{'-root',[<<37 bytes>>]},{'-progname',[<<3 bytes>>]},{'-home',[<<12 bytes"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,290}]}, {heap_size,4181}, {total_heap_size,10946}, {links,[<10463.6.0>,<10463.7.0>,<10463.3.0>]}, {memory,88304}, {message_queue_len,0}, {reductions,131540}, {trap_exit,true}]}, {<10463.3.0>, [{registered_name,erl_prim_loader}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b2847f41d20 (erl_prim_loader:loop/3 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb896f0 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) [\"/opt/membase/1.7.0r/bin/ns_server/ebin\",\"/opt/membase/1.7.0r/bin/ns_server/deps", "y(2) <0.2.0>", "y(3) {state,efile,[],none,#Port<0.2>,infinity,undefined,true,{prim_state,false,undefin", "y(4) infinity"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,331}]}, {heap_size,6765}, {total_heap_size,53133}, {links,[#Port<10463.2>,<10463.0.0>]}, {memory,425760}, {message_queue_len,0}, {reductions,1306313}, {trap_exit,true}]}, {<10463.6.0>, [{registered_name,error_logger}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcaaab8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_log_mf_h,false,{state,\"/var/membase/logs\",10485760,10,1430609,1,{fil", "y(3) error_logger","y(4) <0.2.0>", "0x00002aaaabcaaae8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,30}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10463.0.0>,<10463.25.0>,#Port<10463.981>]}, {memory,54960}, {message_queue_len,0}, {reductions,462208}, {trap_exit,true}]}, {<10463.7.0>, [{registered_name,application_controller}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcc8ae0 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) infinity", "y(2) application_controller", "y(3) {state,[],[],[],[{inets,<0.584.0>},{ns_server,<0.52.0>},{mnesia,<0.70.0>},{os_mon", "y(4) application_controller", "y(5) <0.2.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [<10463.41.0>,<10463.70.0>,<10463.584.0>,<10463.52.0>, <10463.9.0>,<10463.32.0>,<10463.0.0>]}, {memory,263704}, {message_queue_len,0}, {reductions,40425}, {trap_exit,true}]}, {<10463.9.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa62f0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.10.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_ser", "y(2) <0.7.0>", "0x00002b2847fa6310 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10463.7.0>,<10463.10.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<10463.10.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f96790 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) kernel","y(2) <0.11.0>", "y(3) <0.9.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.9.0>,<10463.11.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,72}, {trap_exit,true}]}, {<10463.11.0>, [{registered_name,kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadba358 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_sup},one_for_all,[{child,<0.58.0>,net_sup_dynamic,{erl_distr", "y(4) kernel_sup","y(5) <0.10.0>", "0x00002aaaaadba390 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10463.22.0>,<10463.26.0>,<10463.27.0>,<10463.58.0>, <10463.24.0>,<10463.17.0>,<10463.20.0>,<10463.21.0>, <10463.19.0>,<10463.12.0>,<10463.13.0>,<10463.10.0>]}, {memory,12112}, {message_queue_len,0}, {reductions,1492}, {trap_exit,true}]}, {<10463.12.0>, [{registered_name,rex}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaddbbb78 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) rpc", "y(3) {1,{<0.6296.0>,{<11894.1166.0>,{#Ref<11894.0.0.23940>,'ns_1@10.1.5.192'}},nil,nil", "y(4) rex","y(5) <0.11.0>", "0x00002aaaaddbbbb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,317811}, {total_heap_size,832040}, {links,[<10463.11.0>]}, {memory,6657272}, {message_queue_len,0}, {reductions,4542}, {trap_exit,true}]}, {<10463.13.0>, [{registered_name,global_name_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab03a4d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) global", "y(3) {state,true,['ns_1@10.1.5.193','ns_1@10.1.5.178'],['ns_1@10.1.5.178','ns_1@10.1.5", "y(4) global_name_server","y(5) <0.11.0>", "0x00002aaaab03a508 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,56}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10463.14.0>,<10463.16.0>,<10463.15.0>,<10463.11.0>]}, {memory,11936}, {message_queue_len,0}, {reductions,7610}, {trap_exit,true}]}, {<10463.14.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac82618 (global:loop_the_locker/1 + 1176)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae27c98 Return addr 0x00002aaaaac82140 (global:init_the_locker/1 + 384)", "y(0) {multi,[],[],['ns_1@10.1.5.178','ns_1@10.1.5.193'],'ns_1@10.1.5.193',false,false}", "y(1) infinity", "0x00002aaaaae27cb0 Return addr 0x0000000000816eb8 ()"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10463.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,759}, {trap_exit,true}]}, {<10463.15.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a770 (global:collect_deletions/2 + 152)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa8930 Return addr 0x00002aaaaac8a670 (global:loop_the_deleter/1 + 72)", "y(0) infinity","y(1) []", "y(2) <0.13.0>", "0x00002b2847fa8950 Return addr 0x0000000000816eb8 ()", "y(0) <0.13.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.13.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,4}, {trap_exit,false}]}, {<10463.16.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a9b0 (global:loop_the_registrar/0 + 24)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa1808 Return addr 0x0000000000816eb8 ()", "y(0) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10463.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,248}, {trap_exit,false}]}, {<10463.17.0>, [{registered_name,inet_db}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fd2e20 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) inet_db", "y(3) {state,inet_db,inet_cache,inet_hosts_byname,inet_hosts_byaddr,inet_hosts_file_byn", "y(4) inet_db","y(5) <0.11.0>", "0x00002b2847fd2e58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10463.11.0>]}, {memory,26312}, {message_queue_len,0}, {reductions,1457}, {trap_exit,true}]}, {<10463.19.0>, [{registered_name,global_group}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa75f8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) global_group", "y(3) {state,no_conf,true,[],[],[],[],[],'nonode@nohost',[],normal,normal}", "y(4) global_group","y(5) <0.11.0>", "0x00002b2847fa7630 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,84}, {trap_exit,true}]}, {<10463.20.0>, [{registered_name,file_server_2}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae3ece0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) file_server","y(3) #Port<0.223>", "y(4) file_server_2","y(5) <0.11.0>", "0x00002aaaaae3ed18 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,623}]}, {heap_size,4181}, {total_heap_size,15127}, {links,[#Port<10463.223>,<10463.11.0>]}, {memory,121816}, {message_queue_len,0}, {reductions,832561}, {trap_exit,true}]}, {<10463.21.0>, [{registered_name,code_server}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaad54b40 (code_server:loop/1 + 128)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2848007f08 Return addr 0x0000000000816eb8 ()", "y(0) {state,<0.11.0>,\"/opt/membase/1.7.0r/erlang/lib/erlang\",[\"/opt/membase/1.7.0r/bin", "y(1) <0.11.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,212}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10463.11.0>]}, {memory,284032}, {message_queue_len,0}, {reductions,263571}, {trap_exit,true}]}, {<10463.22.0>, [{registered_name,standard_error_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fc93d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,standard_error,<0.23.0>,<0.23.0>,{local,standard_error_sup}}", "y(4) standard_error_sup","y(5) <0.11.0>", "0x00002b2847fc9408 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.11.0>,<10463.23.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10463.23.0>, [{registered_name,standard_error}, {status,waiting}, {initial_call,{standard_error,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad203a0 (standard_error:server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fca2a8 Return addr 0x0000000000816eb8 ()", "y(0) #Port<0.506>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.22.0>,#Port<10463.506>]}, {memory,2664}, {message_queue_len,0}, {reductions,7}, {trap_exit,true}]}, {<10463.24.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb8478 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,user_sup,<0.25.0>,<0.25.0>,{<0.24.0>,user_sup}}", "y(4) <0.24.0>","y(5) <0.11.0>", "0x00002aaaaadb84b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<10463.11.0>,<10463.25.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,71}, {trap_exit,true}]}, {<10463.25.0>, [{registered_name,user}, {status,waiting}, {initial_call,{user,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad765e0 (user:server_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae51130 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) {[],[]}", "y(3) #Port<0.523>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<10463.6.0>,<10463.24.0>,#Port<10463.523>]}, {memory,109080}, {message_queue_len,0}, {reductions,27428}, {trap_exit,true}]}, {<10463.26.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fc9b20 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) kernel_config","y(3) []", "y(4) <0.26.0>","y(5) <0.11.0>", "0x00002b2847fc9b58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,176}, {trap_exit,true}]}, {<10463.27.0>, [{registered_name,kernel_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb3f88 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_safe_sup},one_for_one,[{child,<0.92.0>,dets,{dets_server,sta", "y(4) kernel_safe_sup","y(5) <0.11.0>", "0x00002aaaaadb3fc0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10463.87.0>,<10463.91.0>,<10463.92.0>,<10463.88.0>, <10463.50.0>,<10463.56.0>,<10463.11.0>]}, {memory,7032}, {message_queue_len,0}, {reductions,439}, {trap_exit,true}]}, {<10463.32.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fab240 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.33.0>,{appl_data,sasl,[sasl_sup,alarm_handler,overload,release_handler]", "y(2) <0.7.0>", "0x00002b2847fab260 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.7.0>,<10463.33.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,27}, {trap_exit,true}]}, {<10463.33.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa3df0 Return addr 0x0000000000816eb8 ()", "y(0) {state,tty,undefined}","y(1) sasl", "y(2) <0.34.0>","y(3) <0.32.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.32.0>,<10463.34.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,70}, {trap_exit,true}]}, {<10463.34.0>, [{registered_name,sasl_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff5ef8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_sup},one_for_one,[{child,<0.38.0>,release_handler,{release_han", "y(4) sasl_sup","y(5) <0.33.0>", "0x00002b2847ff5f30 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.35.0>,<10463.38.0>,<10463.33.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,276}, {trap_exit,true}]}, {<10463.35.0>, [{registered_name,sasl_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f90f50 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_safe_sup},one_for_one,[{child,<0.37.0>,overload,{overload,star", "y(4) sasl_safe_sup","y(5) <0.34.0>", "0x00002b2847f90f88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.36.0>,<10463.37.0>,<10463.34.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,174}, {trap_exit,true}]}, {<10463.36.0>, [{registered_name,alarm_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fca9c8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,alarm_handler,false,[],false}]", "y(3) alarm_handler","y(4) <0.35.0>", "0x00002b2847fca9f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,28}, {trap_exit,true}]}, {<10463.37.0>, [{registered_name,overload}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fcb860 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) overload", "y(3) {state,0,0,8.000000e-01,223,1.000000e-01,{0,0},clear}", "y(4) overload","y(5) <0.35.0>", "0x00002b2847fcb898 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,39}, {trap_exit,false}]}, {<10463.38.0>, [{registered_name,release_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea2440 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) release_handler", "y(3) {state,[],\"/opt/membase/1.7.0r/erlang/lib/erlang\",\"/opt/membase/1.7.0r/erlang/lib", "y(4) release_handler","y(5) <0.34.0>", "0x00002aaaaaea2478 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,4181}, {links,[<10463.34.0>]}, {memory,34208}, {message_queue_len,0}, {reductions,2237}, {trap_exit,false}]}, {<10463.41.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fcb128 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.42.0>,{appl_data,os_mon,[os_mon_sup,os_mon_sysinfo,disksup,memsup,cpu_s", "y(2) <0.7.0>", "0x00002b2847fcb148 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.7.0>,<10463.42.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<10463.42.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa2390 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) os_mon","y(2) <0.43.0>", "y(3) <0.41.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.41.0>,<10463.43.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10463.43.0>, [{registered_name,os_mon_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847faa808 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,os_mon_sup},one_for_one,[{child,<0.47.0>,cpu_sup,{cpu_sup,start_lin", "y(4) os_mon_sup","y(5) <0.42.0>", "0x00002b2847faa840 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10463.44.0>,<10463.45.0>,<10463.47.0>,<10463.42.0>]}, {memory,6912}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10463.44.0>, [{registered_name,disksup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02e468 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) disksup", "y(3) {state,80,60000,{unix,linux},[{\"/\",13140720,42},{\"/boot\",101086,13},{\"/dev/shm\",1", "y(4) disksup","y(5) <0.43.0>", "0x00002aaaab02e4a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,33}]}, {heap_size,2584}, {total_heap_size,5168}, {links,[<10463.43.0>,#Port<10463.879>]}, {memory,42144}, {message_queue_len,0}, {reductions,28601}, {trap_exit,true}]}, {<10463.45.0>, [{registered_name,memsup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1120b0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) memsup", "y(3) {state,{unix,linux},true,{1594331136,2058928128},{<0.462.0>,2172304},false,60000,", "y(4) memsup","y(5) <0.43.0>", "0x00002aaaab1120e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,45}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<10463.43.0>,<10463.46.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,47158}, {trap_exit,true}]}, {<10463.46.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf49870 (memsup:port_idle/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc005f0 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.908>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,37}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10463.45.0>,#Port<10463.908>]}, {memory,16488}, {message_queue_len,0}, {reductions,13188}, {trap_exit,true}]}, {<10463.47.0>, [{registered_name,cpu_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc7e18 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) cpu_sup", "y(3) {state,<0.48.0>,{unix,linux}}", "y(4) cpu_sup","y(5) <0.43.0>", "0x00002aaaaadc7e50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.43.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,33}, {trap_exit,true}]}, {<10463.48.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3a528 (cpu_sup:measurement_server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa2ab8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []","y(5) []", "y(6) []","y(7) []", "y(8) {internal,<0.49.0>,[],{unix,linux}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.49.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,11}, {trap_exit,true}]}, {<10463.49.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3b700 (cpu_sup:port_server_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae296e8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) 6000", "y(2) #Port<0.946>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10463.48.0>,#Port<10463.946>]}, {memory,8592}, {message_queue_len,0}, {reductions,299}, {trap_exit,false}]}, {<10463.50.0>, [{registered_name,timer_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadaeeb8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) 460","y(2) timer", "y(3) []","y(4) timer_server", "y(5) <0.27.0>", "0x00002aaaaadaeef0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2037}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10463.144.0>,<10463.154.0>,<10463.448.0>, <10463.461.0>,<10463.166.0>,<10463.147.0>, <10463.153.0>,<10463.145.0>,<10463.125.0>, <10463.141.0>,<10463.143.0>,<10463.129.0>, <10463.78.0>,<10463.82.0>,<10463.27.0>]}, {memory,12232}, {message_queue_len,0}, {reductions,1392175}, {trap_exit,true}]}, {<10463.52.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9ed30 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.53.0>,{appl_data,ns_server,[ns_server_sup,ns_config,ns_config_sup,ns_co", "y(2) <0.7.0>", "0x00002b2847f9ed50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.7.0>,<10463.53.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,46}, {trap_exit,true}]}, {<10463.53.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9e5e8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) ns_server", "y(2) <0.54.0>","y(3) <0.52.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.52.0>,<10463.54.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,50}, {trap_exit,true}]}, {<10463.54.0>, [{registered_name,ns_server_cluster_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fe5258 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_cluster_sup},one_for_one,[{child,<0.124.0>,ns_server_sup,", "y(4) ns_server_cluster_sup","y(5) <0.53.0>", "0x00002b2847fe5290 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,2584}, {total_heap_size,5168}, {links, [<10463.65.0>,<10463.117.0>,<10463.124.0>, <10463.55.0>,<10463.64.0>,<10463.53.0>]}, {memory,42304}, {message_queue_len,0}, {reductions,1840}, {trap_exit,true}]}, {<10463.55.0>, [{registered_name,dist_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa4520 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dist_manager", "y(3) {state,true,\"10.1.5.192\"}", "y(4) dist_manager","y(5) <0.54.0>", "0x00002b2847fa4558 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.54.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,912}, {trap_exit,false}]}, {<10463.56.0>, [{registered_name,inet_gethost_native_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaeac158 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,inet_gethost_native,<0.57.0>,<0.57.0>,{local,inet_gethost_native_sup}}", "y(4) inet_gethost_native_sup", "y(5) <0.27.0>", "0x00002aaaaaeac190 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.27.0>,<10463.57.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,41}, {trap_exit,true}]}, {<10463.57.0>, [{registered_name,inet_gethost_native}, {status,waiting}, {initial_call,{inet_gethost_native,server_init,2}}, {backtrace, ["Program counter: 0x00002aaaaacc9bd8 (inet_gethost_native:main_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab038628 Return addr 0x0000000000816eb8 ()", "y(0) {state,#Port<0.1012>,8000,8209,12306,<0.56.0>,4,{statistics,0,0,0,0,0,0,0,0}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,625}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10463.56.0>,#Port<10463.1012>]}, {memory,8696}, {message_queue_len,0}, {reductions,52918}, {trap_exit,true}]}, {<10463.58.0>, [{registered_name,net_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fe7d10 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,net_sup},one_for_all,[{child,<0.61.0>,net_kernel,{net_kernel,start_", "y(4) net_sup","y(5) <0.11.0>", "0x00002b2847fe7d48 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links, [<10463.59.0>,<10463.60.0>,<10463.61.0>,<10463.11.0>]}, {memory,8776}, {message_queue_len,0}, {reductions,252}, {trap_exit,true}]}, {<10463.59.0>, [{registered_name,erl_epmd}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcb020 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) erl_epmd", "y(3) {state,#Port<0.1048>,21100,ns_1}", "y(4) erl_epmd","y(5) <0.58.0>", "0x00002aaaaadcb058 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.58.0>,#Port<10463.1048>]}, {memory,2664}, {message_queue_len,0}, {reductions,127}, {trap_exit,false}]}, {<10463.60.0>, [{registered_name,auth}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f90800 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) auth", "y(3) {state,lulxzmudqfmlvwvs,16403}", "y(4) auth","y(5) <0.58.0>", "0x00002b2847f90838 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.58.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,223}, {trap_exit,true}]}, {<10463.61.0>, [{registered_name,net_kernel}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1cc78 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) net_kernel", "y(3) {state,'ns_1@10.1.5.192','ns_1@10.1.5.192',longnames,{tick,<0.63.0>,15000},7000,s", "y(4) net_kernel","y(5) <0.58.0>", "0x00002aaaaae1ccb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,12}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10463.63.0>,<10463.604.0>,<10463.643.0>, <10463.58.0>,<10463.62.0>,#Port<10463.1046>]}, {memory,8888}, {message_queue_len,0}, {reductions,2225}, {trap_exit,true}]}, {<10463.62.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,accept_loop,2}}, {backtrace, ["Program counter: 0x00002b2847f14ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadccd60 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 8538","y(1) #Port<0.1046>", "0x00002aaaaadccd78 Return addr 0x00002aaaaaf84540 (inet_tcp_dist:accept_loop/2 + 96)", "y(0) []", "0x00002aaaaadccd88 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.1046>", "y(2) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.61.0>]}, {memory,2592}, {message_queue_len,0}, {reductions,268}, {trap_exit,false}]}, {<10463.63.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{net_kernel,ticker,2}}, {backtrace, ["Program counter: 0x00002aaaaad358c8 (net_kernel:ticker_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcbef0 Return addr 0x0000000000816eb8 ()", "y(0) 15000","y(1) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.61.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,161}, {trap_exit,false}]}, {<10463.64.0>, [{registered_name,ns_cluster}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc7adb8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_cluster","y(3) {state}", "y(4) ns_cluster","y(5) <0.54.0>", "0x00002aaaabc7adf0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,17711}, {total_heap_size,46368}, {links,[<10463.54.0>]}, {memory,371704}, {message_queue_len,0}, {reductions,21119}, {trap_exit,false}]}, {<10463.65.0>, [{registered_name,mb_mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9de78 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mb_mnesia_sup},one_for_one,[{child,<0.67.0>,mb_mnesia,{mb_mnesia,st", "y(4) mb_mnesia_sup","y(5) <0.54.0>", "0x00002b2847f9deb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.66.0>,<10463.67.0>,<10463.54.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<10463.66.0>, [{registered_name,mb_mnesia_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcd4b8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) mb_mnesia_events","y(4) <0.65.0>", "0x00002aaaaadcd4e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.65.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,32}, {trap_exit,true}]}, {<10463.67.0>, [{registered_name,mb_mnesia}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0ea650 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_mnesia", "y(3) {state,['ns_1@10.1.5.192']}", "y(4) mb_mnesia","y(5) <0.65.0>", "0x00002aaaab0ea688 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10463.65.0>,<10463.76.0>]}, {memory,54920}, {message_queue_len,0}, {reductions,4622}, {trap_exit,true}]}, {<10463.70.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9bfd8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.71.0>,{appl_data,mnesia,[mnesia_dumper_load_regulator,mnesia_event,mnes", "y(2) <0.7.0>", "0x00002b2847f9bff8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.7.0>,<10463.71.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<10463.71.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadce370 Return addr 0x0000000000816eb8 ()", "y(0) {normal,[]}","y(1) mnesia_sup", "y(2) <0.72.0>","y(3) <0.70.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.70.0>,<10463.72.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10463.72.0>, [{registered_name,mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1dad8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_sup},one_for_all,[{child,<0.74.0>,mnesia_kernel_sup,{mnesia_", "y(4) mnesia_sup","y(5) <0.71.0>", "0x00002aaaaae1db10 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10463.73.0>,<10463.74.0>,<10463.71.0>]}, {memory,6872}, {message_queue_len,0}, {reductions,207}, {trap_exit,true}]}, {<10463.73.0>, [{registered_name,mnesia_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9b878 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,mnesia_event,false,{state,[],false,[]},false}]", "y(3) mnesia_event","y(4) <0.72.0>", "0x00002b2847f9b8a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10463.72.0>,<10463.76.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,408}, {trap_exit,true}]}, {<10463.74.0>, [{registered_name,mnesia_kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9ac98 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_kernel_sup},one_for_all,[{child,<0.83.0>,mnesia_late_loader,", "y(4) mnesia_kernel_sup","y(5) <0.72.0>", "0x00002b2847f9acd0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10463.77.0>,<10463.81.0>,<10463.82.0>,<10463.83.0>, <10463.79.0>,<10463.80.0>,<10463.78.0>,<10463.75.0>, <10463.76.0>,<10463.72.0>]}, {memory,9016}, {message_queue_len,0}, {reductions,599}, {trap_exit,true}]}, {<10463.75.0>, [{registered_name,mnesia_monitor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae53bc0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_monitor", "y(3) {state,<0.74.0>,[],[],true,[],undefined,[]}", "y(4) mnesia_monitor","y(5) <0.74.0>", "0x00002aaaaae53bf8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,37}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10463.89.0>,<10463.74.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,7361}, {trap_exit,true}]}, {<10463.76.0>, [{registered_name,mnesia_subscr}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcf670 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_subscr", "y(3) {state,<0.74.0>,20503}", "y(4) mnesia_subscr","y(5) <0.74.0>", "0x00002aaaaadcf6a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.73.0>,<10463.74.0>,<10463.67.0>]}, {memory,2704}, {message_queue_len,0}, {reductions,111}, {trap_exit,true}]}, {<10463.77.0>, [{registered_name,mnesia_locker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab3ac880 (mnesia_locker:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae3ffc8 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []", "y(5) {state,<0.74.0>}", "0x00002aaaaae40000 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_locker","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002aaaaae40030 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,598}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10463.74.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,320197}, {trap_exit,true}]}, {<10463.78.0>, [{registered_name,mnesia_recover}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdc5f18 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_recover", "y(3) {state,<0.74.0>,undefined,undefined,undefined,0,true,[]}", "y(4) mnesia_recover","y(5) <0.74.0>", "0x00002aaaabdc5f50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10463.74.0>,<10463.50.0>]}, {memory,11712}, {message_queue_len,0}, {reductions,2930}, {trap_exit,true}]}, {<10463.79.0>, [{registered_name,mnesia_tm}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab3bcea0 (mnesia_tm:doit_loop/1 + 216)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadbe820 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,{0,nil},{0,nil},<0.74.0>,[],[],[]}", "y(3) []","y(4) []","y(5) <0.74.0>", "y(6) {0,nil}","y(7) {0,nil}", "0x00002aaaaadbe868 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_tm","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002aaaaadbe898 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,248}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10463.74.0>]}, {memory,16552}, {message_queue_len,0}, {reductions,744361}, {trap_exit,true}]}, {<10463.80.0>, [{registered_name,mnesia_checkpoint_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f981e0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_checkpoint_sup},simple_one_for_one,[{child,undefined,mnesia_", "y(4) mnesia_checkpoint_sup","y(5) <0.74.0>", "0x00002b2847f98218 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10463.81.0>, [{registered_name,mnesia_snmp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa3210 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_snmp_sup},simple_one_for_one,[{child,undefined,mnesia_snmp_s", "y(4) mnesia_snmp_sup","y(5) <0.74.0>", "0x00002b2847fa3248 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10463.82.0>, [{registered_name,mnesia_controller}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadca8d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_controller", "y(3) {state,<0.74.0>,true,[],[],{0,nil},[],[],{0,nil},undefined,[],[],{interval,#Ref<0", "y(4) mnesia_controller","y(5) <0.74.0>", "0x00002aaaaadca908 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10463.74.0>,<10463.50.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,937}, {trap_exit,true}]}, {<10463.83.0>, [{registered_name,mnesia_late_loader}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab3a8d68 (mnesia_late_loader:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f98d98 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,<0.74.0>}", "0x00002b2847f98db8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_late_loader","y(2) []", "y(3) []","y(4) [<0.74.0>]", "0x00002b2847f98de8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,377}, {total_heap_size,377}, {links,[<10463.74.0>]}, {memory,3776}, {message_queue_len,0}, {reductions,172}, {trap_exit,false}]}, {<10463.87.0>, [{registered_name,disk_log_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab09d090 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,disk_log_sup},simple_one_for_one,[{child,undefined,disk_log,{disk_l", "y(4) disk_log_sup","y(5) <0.27.0>", "0x00002aaaab09d0c8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<10463.89.0>,<10463.27.0>]}, {memory,26352}, {message_queue_len,0}, {reductions,4499}, {trap_exit,true}]}, {<10463.88.0>, [{registered_name,disk_log_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1101d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) disk_log_server","y(3) {state,[]}", "y(4) disk_log_server","y(5) <0.27.0>", "0x00002aaaab110208 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,44}]}, {heap_size,1597}, {total_heap_size,2584}, {links,[<10463.89.0>,<10463.27.0>]}, {memory,21552}, {message_queue_len,0}, {reductions,4613}, {trap_exit,true}]}, {<10463.89.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab440f40 (disk_log:loop/1 + 168)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fbd688 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) {state,[],[],<0.87.0>,<0.88.0>,596,{arg,latest_log,undefined,\"/var/membase/mnesia", "0x00002b2847fbd698 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,2584}, {total_heap_size,4181}, {links, [<10463.87.0>,<10463.88.0>,<10463.75.0>, #Port<10463.5368>]}, {memory,34328}, {message_queue_len,0}, {reductions,217111}, {trap_exit,true}]}, {<10463.91.0>, [{registered_name,dets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1a1c0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,dets_sup},simple_one_for_one,[{child,undefined,dets,{dets,istart_li", "y(4) dets_sup","y(5) <0.27.0>", "0x00002aaaaae1a1f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10463.27.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,1139}, {trap_exit,true}]}, {<10463.92.0>, [{registered_name,dets}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9c710 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dets_server", "y(3) {state,40996,[<0.27.0>],[]}", "y(4) dets","y(5) <0.27.0>", "0x00002b2847f9c748 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,35}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.27.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,1756}, {trap_exit,true}]}, {<10463.117.0>, [{registered_name,ns_config_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab015c50 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_config_sup},rest_for_one,[{child,undefined,ns_config_log,{ns_con", "y(4) ns_config_sup","y(5) <0.54.0>", "0x00002aaaab015c88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10463.118.0>,<10463.119.0>,<10463.54.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,580}, {trap_exit,true}]}, {<10463.118.0>, [{registered_name,ns_config_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc3948 Return addr 0x00002b2847f5d018 (proc_lib:wake_up/3 + 120)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.680>,{state,#Fun,ok},<0.17", "y(3) ns_config_events","y(4) <0.117.0>", "0x00002aaaaadc3978 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5d038 (proc_lib:wake_up/3 + 152)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,24}]}, {heap_size,610}, {total_heap_size,1597}, {links, [<10463.141.0>,<10463.164.0>,<10463.172.0>, <10463.117.0>]}, {memory,13872}, {message_queue_len,0}, {reductions,387411}, {trap_exit,true}]}, {<10463.119.0>, [{registered_name,ns_config}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac169af0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config", "y(3) {config,{full,\"/etc/membase/priv/config\",undefined,ns_config_default},[[{director", "y(4) ns_config","y(5) <0.117.0>", "0x00002aaaac169b28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,46368}, {total_heap_size,92736}, {links,[<10463.117.0>]}, {memory,742648}, {message_queue_len,0}, {reductions,149860}, {trap_exit,true}]}, {<10463.124.0>, [{registered_name,ns_server_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab819868 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_sup},one_for_one,[{child,<0.232.0>,moxi_stats_collector,{", "y(4) ns_server_sup","y(5) <0.54.0>", "0x00002aaaab8198a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,75025}, {total_heap_size,196418}, {links, [<10463.149.0>,<10463.163.0>,<10463.171.0>, <10463.172.0>,<10463.232.0>,<10463.165.0>, <10463.166.0>,<10463.164.0>,<10463.161.0>, <10463.162.0>,<10463.155.0>,<10463.127.0>, <10463.145.0>,<10463.147.0>,<10463.141.0>, <10463.125.0>,<10463.126.0>,<10463.54.0>]}, {memory,1572784}, {message_queue_len,0}, {reductions,87934}, {trap_exit,true}]}, {<10463.125.0>, [{registered_name,ns_log}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae36a30 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) ns_log", "y(3) {state,[{log_entry,{1302,303452,740950},'ns_1@10.1.5.192',ns_node_disco,3,\"Initia", "y(4) ns_log","y(5) <0.124.0>", "0x00002aaaaae36a68 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<10463.50.0>,<10463.124.0>]}, {memory,109040}, {message_queue_len,0}, {reductions,6608}, {trap_exit,false}]}, {<10463.126.0>, [{registered_name,ns_log_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae923e0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) ns_log_events","y(4) <0.124.0>", "0x00002aaaaae92410 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,1597}, {links,[<10463.124.0>]}, {memory,13536}, {message_queue_len,0}, {reductions,88}, {trap_exit,true}]}, {<10463.127.0>, [{registered_name,ns_node_disco_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae93df0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_node_disco_sup},rest_for_one,[{child,<0.136.0>,ns_config_rep,{ns", "y(4) ns_node_disco_sup","y(5) <0.124.0>", "0x00002aaaaae93e28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10463.128.0>,<10463.129.0>,<10463.136.0>, <10463.124.0>]}, {memory,6912}, {message_queue_len,0}, {reductions,829}, {trap_exit,true}]}, {<10463.128.0>, [{registered_name,ns_node_disco_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdc0e50 Return addr 0x00002b2847f5d018 (proc_lib:wake_up/3 + 120)", "y(0) false","y(1) []", "y(2) [{handler,menelaus_event,ns_node_disco_events,{state,ns_node_disco_events,undefin", "y(3) ns_node_disco_events","y(4) <0.127.0>", "0x00002aaaabdc0e80 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5d038 (proc_lib:wake_up/3 + 152)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,44}]}, {heap_size,377}, {total_heap_size,521}, {links,[<10463.127.0>]}, {memory,5144}, {message_queue_len,0}, {reductions,2817}, {trap_exit,true}]}, {<10463.129.0>, [{registered_name,ns_node_disco}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac0ec870 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_node_disco", "y(3) {state,['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193'],{interval,#Ref<0.0", "y(4) ns_node_disco","y(5) <0.127.0>", "0x00002aaaac0ec8a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,566}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10463.127.0>,<10463.50.0>]}, {memory,601000}, {message_queue_len,0}, {reductions,42950}, {trap_exit,false}]}, {<10463.136.0>, [{registered_name,ns_config_rep}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaceb68e0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config_rep","y(3) {state}", "y(4) ns_config_rep","y(5) <0.127.0>", "0x00002aaaaceb6918 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,46368}, {total_heap_size,57314}, {links,[<10463.127.0>]}, {memory,459272}, {message_queue_len,0}, {reductions,18821}, {trap_exit,false}]}, {<10463.141.0>, [{registered_name,mb_master}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab59a340 (gen_fsm:loop/7 + 312)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab193850 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_master", "y(3) {state,<0.142.0>,'ns_1@10.1.5.192',['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.", "y(4) master","y(5) mb_master", "y(6) <0.124.0>", "0x00002aaaab193890 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,111}]}, {heap_size,1597}, {total_heap_size,1974}, {links, [<10463.118.0>,<10463.124.0>,<10463.142.0>, <10463.50.0>]}, {memory,16672}, {message_queue_len,0}, {reductions,39233}, {trap_exit,false}]}, {<10463.142.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa9068 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{<0.142.0>,mb_master_sup},one_for_one,[{child,<0.144.0>,ns_tick,{ns_tick,s", "y(4) <0.142.0>","y(5) <0.141.0>", "0x00002b2847fa90a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.143.0>,<10463.144.0>,<10463.141.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,197}, {trap_exit,true}]}, {<10463.143.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab59a340 (gen_fsm:loop/7 + 312)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac319218 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_orchestrator", "y(3) {idle_state,[]}","y(4) idle", "y(5) ns_orchestrator","y(6) <0.142.0>", "0x00002aaaac319258 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,31}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10463.142.0>,<10463.50.0>]}, {memory,601312}, {message_queue_len,0}, {reductions,2346854}, {trap_exit,true}]}, {<10463.144.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab033530 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) ns_tick", "y(3) {state,1302304649807}","y(4) ns_tick", "y(5) <0.142.0>", "0x00002aaaab033568 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,72}]}, {heap_size,2584}, {total_heap_size,2961}, {links,[<10463.142.0>,<10463.50.0>]}, {memory,24800}, {message_queue_len,0}, {reductions,46899}, {trap_exit,false}]}, {<10463.145.0>, [{registered_name,ns_heart}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fb8598 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_heart", "y(3) [{meminfo,<<777 bytes>>},{replication,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9c", "y(4) ns_heart","y(5) <0.124.0>", "0x00002b2847fb85d0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10463.124.0>,<10463.50.0>]}, {memory,284176}, {message_queue_len,0}, {reductions,1790261}, {trap_exit,false}]}, {<10463.147.0>, [{registered_name,ns_doctor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab137c08 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_doctor", "y(3) {state,{dict,3,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{", "y(4) ns_doctor","y(5) <0.124.0>", "0x00002aaaab137c40 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10463.124.0>,<10463.50.0>]}, {memory,142488}, {message_queue_len,0}, {reductions,88846}, {trap_exit,false}]}, {<10463.149.0>, [{registered_name,menelaus_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f900b0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,menelaus_sup},one_for_one,[{child,<0.154.0>,menelaus_web_alerts_srv", "y(4) menelaus_sup","y(5) <0.124.0>", "0x00002b2847f900e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,2584}, {total_heap_size,13530}, {links, [<10463.150.0>,<10463.153.0>,<10463.154.0>, <10463.124.0>]}, {memory,109120}, {message_queue_len,0}, {reductions,1870}, {trap_exit,true}]}, {<10463.150.0>, [{registered_name,menelaus_web}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1430a0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mochiweb_socket_server", "y(3) {mochiweb_socket_server,8091,#Fun,{local,menelaus_web}", "y(4) menelaus_web","y(5) <0.149.0>", "0x00002aaaab1430d8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,1597}, {total_heap_size,1974}, {links, [<10463.450.0>,<10463.3204.0>,<10463.6295.0>, <10463.2695.0>,<10463.149.0>,<10463.224.0>, #Port<10463.2158>]}, {memory,16792}, {message_queue_len,0}, {reductions,2738}, {trap_exit,true}]}, {<10463.153.0>, [{registered_name,hot_keys_keeper}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab18d478 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) hot_keys_keeper", "y(3) {state,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",[{\"key_eb13eea", "y(4) hot_keys_keeper","y(5) <0.149.0>", "0x00002aaaab18d4b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,94}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10463.149.0>,<10463.50.0>]}, {memory,142488}, {message_queue_len,0}, {reductions,7406}, {trap_exit,false}]}, {<10463.154.0>, [{registered_name,menelaus_web_alerts_srv}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea80d8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) menelaus_web_alerts_srv", "y(3) {state,[],[],{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],", "y(4) menelaus_web_alerts_srv", "y(5) <0.149.0>", "0x00002aaaaaea8110 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,684}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10463.50.0>,<10463.149.0>]}, {memory,54920}, {message_queue_len,0}, {reductions,409187}, {trap_exit,false}]}, {<10463.155.0>, [{registered_name,ns_port_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac1aedf8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_port_sup},one_for_one,[{child,<0.222.0>,{moxi,\"./bin/moxi/moxi\",", "y(4) ns_port_sup","y(5) <0.124.0>", "0x00002aaaac1aee30 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10463.222.0>,<10463.428.0>,<10463.124.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,4608}, {trap_exit,true}]}, {<10463.161.0>, [{registered_name,ns_tick_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f96ec8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7737>,{state,#Fun,ignored},<", "y(3) ns_tick_event","y(4) <0.124.0>", "0x00002b2847f96ef8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,258}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.165.0>,<10463.460.0>,<10463.124.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,19681}, {trap_exit,true}]}, {<10463.162.0>, [{registered_name,ns_stats_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab075660 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7761>,{state,#Fun,ignored},<", "y(3) ns_stats_event","y(4) <0.124.0>", "0x00002aaaab075690 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,505}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10463.166.0>,<10463.461.0>,<10463.124.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,75250}, {trap_exit,true}]}, {<10463.163.0>, [{registered_name,ns_bucket_worker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b284800a988 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) work_queue","y(3) []", "y(4) ns_bucket_worker","y(5) <0.124.0>", "0x00002b284800a9c0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10463.124.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,394}, {trap_exit,false}]}, {<10463.164.0>, [{registered_name,ns_bucket_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0cff70 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_bucket_sup},one_for_one,[{child,<0.446.0>,{per_bucket_sup,\"Failo", "y(4) ns_bucket_sup","y(5) <0.124.0>", "0x00002aaaab0cffa8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10463.124.0>,<10463.446.0>,<10463.118.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,216}, {trap_exit,true}]}, {<10463.165.0>, [{registered_name,system_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff9848 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) system_stats_collector", "y(3) {state,#Port<0.2218>,[{cpu_local_ms,1967642270},{cpu_idle_ms,1843311180},{swap_to", "y(4) system_stats_collector", "y(5) <0.124.0>", "0x00002b2847ff9880 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,306}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10463.124.0>,<10463.161.0>,#Port<10463.2218>]}, {memory,11752}, {message_queue_len,0}, {reductions,69927}, {trap_exit,false}]}, {<10463.166.0>, [{registered_name,'stats_archiver-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fc0858 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"@system\"}", "y(4) 'stats_archiver-@system'", "y(5) <0.124.0>", "0x00002b2847fc0890 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,183}]}, {heap_size,1597}, {total_heap_size,4181}, {links,[<10463.124.0>,<10463.162.0>,<10463.50.0>]}, {memory,34288}, {message_queue_len,0}, {reductions,6090199}, {trap_exit,false}]}, {<10463.171.0>, [{registered_name,'stats_reader-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff6648 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"@system\"}", "y(4) 'stats_reader-@system'", "y(5) <0.124.0>", "0x00002b2847ff6680 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.124.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,26}, {trap_exit,false}]}, {<10463.172.0>, [{registered_name,ns_moxi_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc0ec38 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_moxi_sup},one_for_one,[{child,<0.441.0>,{\"FailoverNodesTests-4a4", "y(4) ns_moxi_sup","y(5) <0.124.0>", "0x00002aaaabc0ec70 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10463.124.0>,<10463.441.0>,<10463.118.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,1343}, {trap_exit,true}]}, {<10463.222.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb1968 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi,5000,{1302,303457,478273},<0.223.0>}", "y(4) <0.222.0>","y(5) <0.155.0>", "0x00002aaaaadb19a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10463.155.0>,<10463.223.0>]}, {memory,21472}, {message_queue_len,0}, {reductions,150}, {trap_exit,true}]}, {<10463.223.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab06b4c8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2338>,moxi,{[\"2011-04-08 15:57:37: (cproxy_config.c.325) env: MOXI", "y(4) <0.223.0>","y(5) <0.222.0>", "0x00002aaaab06b500 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10463.222.0>,#Port<10463.2338>]}, {memory,26352}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10463.224.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab5fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabd00a08 Return addr 0x00002aaaab5f9b78 (menelaus_web:loop/3 + 26376)", "y(0) {struct,[{buckets,[]}]}", "y(1) {struct,[{buckets,[]}]}", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBuck", "y(3) {mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(4) #Fun", "0x00002aaaabd00a38 Return addr 0x00002aaaab5d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(5) Catch 0x00002aaaab5f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaabd00a70 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "0x00002aaaabd00a98 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,28657}, {total_heap_size,28657}, {links,[<10463.150.0>,#Port<10463.2342>]}, {memory,230200}, {message_queue_len,0}, {reductions,7163}, {trap_exit,false}]}, {<10463.232.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ffab68 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi_stats_collector,5000,{1302,303457,921207},<0.233.0>}", "y(4) <0.232.0>","y(5) <0.124.0>", "0x00002b2847ffaba0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.124.0>,<10463.233.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<10463.233.0>, [{registered_name,moxi_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0c2bc0 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) []","y(1) infinity", "y(2) moxi_stats_collector", "y(3) {state,#Port<0.2357>,{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[", "y(4) <0.233.0>","y(5) <0.232.0>", "0x00002aaaab0c2bf8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) moxi_stats_collector","y(2) []", "y(3) moxi_stats_collector", "y(4) {local,moxi_stats_collector}", "y(5) <0.232.0>","y(6) <0.232.0>", "0x00002aaaab0c2c38 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,15}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [#Port<10463.2603>,<10463.232.0>,#Port<10463.2357>]}, {memory,263544}, {message_queue_len,0}, {reductions,11863321}, {trap_exit,false}]}, {<10463.428.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0372d8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,memcached,5000,{1302,303773,930402},<0.429.0>}", "y(4) <0.428.0>","y(5) <0.155.0>", "0x00002aaaab037310 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10463.155.0>,<10463.429.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,55}, {trap_exit,true}]}, {<10463.429.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea9fb8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2527>,memcached,{[empty],[empty,empty]},undefined,[],0}", "y(4) <0.429.0>","y(5) <0.428.0>", "0x00002aaaaaea9ff0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10463.428.0>,#Port<10463.2527>]}, {memory,8696}, {message_queue_len,0}, {reductions,58}, {trap_exit,true}]}, {<10463.441.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0a4038 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2539>,moxi,{[\"ERROR: could not contact REST server(s): http://127.", "y(4) <0.441.0>","y(5) <0.172.0>", "0x00002aaaab0a4070 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10463.172.0>,#Port<10463.2539>]}, {memory,21472}, {message_queue_len,0}, {reductions,356}, {trap_exit,true}]}, {<10463.446.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaab3fe9a0 (single_bucket_sup:top_loop/3 + 72)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9ce80 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "y(2) <0.447.0>","y(3) <0.164.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.164.0>,<10463.447.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,31}, {trap_exit,true}]}, {<10463.447.0>, [{registered_name, 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab03b7e8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95", "y(4) 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.446.0>", "0x00002aaaab03b820 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,1220}, {links, [<10463.460.0>,<10463.461.0>,<10463.462.0>, <10463.448.0>,<10463.459.0>,<10463.446.0>]}, {memory,10720}, {message_queue_len,0}, {reductions,869}, {trap_exit,true}]}, {<10463.448.0>, [{registered_name, 'ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbfc7e8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_memcached", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",#Port<0.2563>}", "y(4) <0.448.0>","y(5) <0.447.0>", "0x00002aaaabbfc820 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,31}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10463.50.0>,<10463.447.0>,#Port<10463.2563>]}, {memory,284216}, {message_queue_len,0}, {reductions,31718629}, {trap_exit,true}]}, {<10463.450.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab5fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabd23358 Return addr 0x00002aaaab65fd20 (menelaus_web_buckets:checking_bucket_ac", "y(0) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(1) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsS", "y(3) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) #Fun", "0x00002aaaabd23388 Return addr 0x00002aaaab5f9a28 (menelaus_web:loop/3 + 26040)", "y(0) #Ref<0.0.0.7709>", "y(1) Catch 0x00002aaaab65fd80 (menelaus_web_buckets:checking_bucket_access/4 + 616)", "y(2) []", "y(3) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) []", "0x00002aaaabd233b8 Return addr 0x00002aaaab5d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(5) Catch 0x00002aaaab5f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaabd233f0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "0x00002aaaabd23418 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,17711}, {total_heap_size,92736}, {links,[<10463.150.0>,#Port<10463.2574>]}, {memory,742832}, {message_queue_len,0}, {reductions,1504567}, {trap_exit,false}]}, {<10463.459.0>, [{registered_name, 'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0a2158 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(4) 'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.447.0>", "0x00002aaaab0a2190 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10463.447.0>]}, {memory,54880}, {message_queue_len,0}, {reductions,1281}, {trap_exit,true}]}, {<10463.460.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab13ccd0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_collector", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",{[157,8053131,27", "y(4) <0.460.0>","y(5) <0.447.0>", "0x00002aaaab13cd08 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,2584}, {total_heap_size,20295}, {links,[<10463.447.0>,<10463.161.0>]}, {memory,163160}, {message_queue_len,0}, {reductions,20390529}, {trap_exit,false}]}, {<10463.461.0>, [{registered_name, 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0f79c0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"}", "y(4) 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.447.0>", "0x00002aaaab0f79f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,430}]}, {heap_size,6765}, {total_heap_size,53133}, {links,[<10463.162.0>,<10463.447.0>,<10463.50.0>]}, {memory,425904}, {message_queue_len,0}, {reductions,4768175}, {trap_exit,false}]}, {<10463.462.0>, [{registered_name, 'stats_reader-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaad5029f0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"}", "y(4) 'stats_reader-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.447.0>", "0x00002aaaad502a28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,196418}, {total_heap_size,271443}, {links,[<10463.447.0>]}, {memory,2172304}, {message_queue_len,0}, {reductions,595219}, {trap_exit,false}]}, {<10463.584.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb2ca0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.585.0>,{appl_data,inets,[inets_sup,httpc_manager],undefined,{inets_app,", "y(2) <0.7.0>", "0x00002aaaaadb2cc0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.7.0>,<10463.585.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<10463.585.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab067eb0 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) inets_app", "y(2) <0.586.0>","y(3) <0.584.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.584.0>,<10463.586.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,42}, {trap_exit,true}]}, {<10463.586.0>, [{registered_name,inets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae99a88 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,inets_sup},one_for_one,[{child,<0.593.0>,tftp_sup,{tftp_sup,start_l", "y(4) inets_sup","y(5) <0.585.0>", "0x00002aaaaae99ac0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10463.587.0>,<10463.592.0>,<10463.593.0>, <10463.588.0>,<10463.585.0>]}, {memory,6952}, {message_queue_len,0}, {reductions,320}, {trap_exit,true}]}, {<10463.587.0>, [{registered_name,ftp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae910c0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ftp_sup},simple_one_for_one,[{child,undefined,undefined,{ftp,start_", "y(4) ftp_sup","y(5) <0.586.0>", "0x00002aaaaae910f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.586.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,54}, {trap_exit,true}]}, {<10463.588.0>, [{registered_name,httpc_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadbefb0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpc_sup},one_for_one,[{child,<0.591.0>,httpc_handler_sup,{httpc_h", "y(4) httpc_sup","y(5) <0.586.0>", "0x00002aaaaadbefe8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10463.589.0>,<10463.591.0>,<10463.586.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,175}, {trap_exit,true}]}, {<10463.589.0>, [{registered_name,httpc_profile_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae43940 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpc_profile_sup},one_for_one,[{child,<0.590.0>,httpc_manager,{htt", "y(4) httpc_profile_sup","y(5) <0.588.0>", "0x00002aaaaae43978 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.588.0>,<10463.590.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,124}, {trap_exit,true}]}, {<10463.590.0>, [{registered_name,httpc_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae0c9b0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) httpc_manager", "y(3) {state,[],4374581,{undefined,4378678},httpc_manager_session_db,httpc_manager,{opt", "y(4) httpc_manager","y(5) <0.589.0>", "0x00002aaaaae0c9e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10463.589.0>]}, {memory,54880}, {message_queue_len,0}, {reductions,615}, {trap_exit,true}]}, {<10463.591.0>, [{registered_name,httpc_handler_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcc0820 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpc_handler_sup},simple_one_for_one,[{child,undefined,undefined,{", "y(4) httpc_handler_sup","y(5) <0.588.0>", "0x00002aaaabcc0858 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10463.588.0>]}, {memory,54880}, {message_queue_len,0}, {reductions,177}, {trap_exit,true}]}, {<10463.592.0>, [{registered_name,httpd_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab144b10 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpd_sup},one_for_one,[],{dict,0,16,16,8,80,48,{[],[],[],[],[],[],", "y(4) httpd_sup","y(5) <0.586.0>", "0x00002aaaab144b48 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.586.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<10463.593.0>, [{registered_name,tftp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae55628 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,tftp_sup},one_for_one,[],{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[", "y(4) tftp_sup","y(5) <0.586.0>", "0x00002aaaaae55660 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.586.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,46}, {trap_exit,true}]}, {<10463.604.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_accept,6}}, {backtrace, ["Program counter: 0x00002aaaabedccd8 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdc4028 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,4237,8565,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.192'", "y(6) {net_address,{{10,1,5,193},38457},\"10.1.5.193\",tcp,inet}", "y(7) #Port<0.3452>", "y(8) 'ns_1@10.1.5.193'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10463.61.0>,#Port<10463.3452>]}, {memory,16488}, {message_queue_len,0}, {reductions,1770}, {trap_exit,false}]}, {<10463.643.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_setup,6}}, {backtrace, ["Program counter: 0x00002aaaabedccd8 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab077528 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,4338,9024,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.192'", "y(6) {net_address,{{10,1,5,178},21100},\"10.1.5.178\",tcp,inet}", "y(7) #Port<0.3471>", "y(8) 'ns_1@10.1.5.178'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10463.61.0>,#Port<10463.3471>]}, {memory,11608}, {message_queue_len,0}, {reductions,1735}, {trap_exit,false}]}, {<10463.2695.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab5fb130 (menelaus_web:handle_pool_info_wait/6 + 280)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc53eb0 Return addr 0x00002aaaab5fabb0 (menelaus_web:check_and_handle_pool_info", "y(0) \"129590282\"","y(1) \"129590282\"", "y(2) 20000","y(3) \"10.1.5.192\"", "y(4) {\"Administrator\",\"password\"}", "y(5) \"default\"", "y(6) {mochiweb_request,#Port<0.5122>,'GET',\"/pools/default?waitChange=20000&etag=12959", "0x00002aaaabc53ef0 Return addr 0x00002aaaab5f9b08 (menelaus_web:loop/3 + 26264)", "y(0) Catch 0x00002aaaab5fac08 (menelaus_web:check_and_handle_pool_info/2 + 336)", "y(1) []","y(2) <0.3203.0>", "0x00002aaaabc53f10 Return addr 0x00002aaaab5d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.5122>,'GET',\"/pools/default?waitChange=20000&etag=12959", "y(5) Catch 0x00002aaaab5f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaabc53f48 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.5122>,'GET',\"/pools/default?waitChange=20000&etag=12959", "0x00002aaaabc53f70 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,17711}, {total_heap_size,46368}, {links, [<10463.150.0>,<10463.3203.0>,#Port<10463.5122>]}, {memory,371928}, {message_queue_len,0}, {reductions,938540}, {trap_exit,false}]}, {<10463.3203.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaab500158 (diag_handler:'-arm_timeout/2-fun-0-'/3 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff6dc0 Return addr 0x0000000000816eb8 ()", "y(0) <0.2695.0>", "y(1) #Fun", "y(2) 23000"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.2695.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,2}, {trap_exit,false}]}, {<10463.3204.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f16000 (prim_inet:recv0/3 + 224)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabca59f0 Return addr 0x00002aaaab5d1310 (mochiweb_http:request/2 + 120)", "y(0) 3844","y(1) #Port<0.8621>", "0x00002aaaabca5a08 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) #Port<0.8621>", "0x00002aaaabca5a20 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,10946}, {total_heap_size,28657}, {links,[<10463.150.0>,#Port<10463.8621>]}, {memory,230056}, {message_queue_len,0}, {reductions,30579}, {trap_exit,false}]}, {<10463.6295.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f14ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcb760 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 3828","y(1) #Port<0.2158>", "0x00002aaaaadcb778 Return addr 0x00002aaaab5dfe88 (mochiweb_socket_server:acceptor_loop/1 ", "y(0) []", "0x00002aaaaadcb788 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) <0.150.0>", "y(2) Catch 0x00002aaaab5dfe88 (mochiweb_socket_server:acceptor_loop/1 + 160)", "0x00002aaaaadcb7a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10463.150.0>]}, {memory,2696}, {message_queue_len,0}, {reductions,38}, {trap_exit,false}]}, {<10463.6296.0>, [{registered_name,[]}, {status,running}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b2847e998d8 (unknown function)", "CP: 0x00002aaaab4fe068 (diag_handler:grab_process_info/1 + 64)", "0x00002aaaac285c98 Return addr 0x00002aaaab500610 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []", "0x00002aaaac285ca8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []","y(1) <0.6296.0>", "0x00002aaaac285cc0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.6295.0>", "0x00002aaaac285cd8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.3204.0>", "0x00002aaaac285cf0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.3203.0>", "0x00002aaaac285d08 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.2695.0>", "0x00002aaaac285d20 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_setup,6}},", "y(1) <0.643.0>", "0x00002aaaac285d38 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_accept,6}}", "y(1) <0.604.0>", "0x00002aaaac285d50 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,tftp_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.593.0>", "0x00002aaaac285d68 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpd_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.592.0>", "0x00002aaaac285d80 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_handler_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.591.0>", "0x00002aaaac285d98 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_manager},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.590.0>", "0x00002aaaac285db0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_profile_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.589.0>", "0x00002aaaac285dc8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.588.0>", "0x00002aaaac285de0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ftp_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.587.0>", "0x00002aaaac285df8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.586.0>", "0x00002aaaac285e10 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.585.0>", "0x00002aaaac285e28 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.584.0>", "0x00002aaaac285e40 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_reader-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) <0.462.0>", "0x00002aaaac285e58 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b9", "y(1) <0.461.0>", "0x00002aaaac285e70 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.460.0>", "0x00002aaaac285e88 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95820", "y(1) <0.459.0>", "0x00002aaaac285ea0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.450.0>", "0x00002aaaac285eb8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) <0.448.0>", "0x00002aaaac285ed0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-5", "y(1) <0.447.0>", "0x00002aaaac285ee8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.446.0>", "0x00002aaaac285f00 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.441.0>", "0x00002aaaac285f18 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.429.0>", "0x00002aaaac285f30 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.428.0>", "0x00002aaaac285f48 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,moxi_stats_collector},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.233.0>", "0x00002aaaac285f60 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.232.0>", "0x00002aaaac285f78 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.224.0>", "0x00002aaaac285f90 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.223.0>", "0x00002aaaac285fa8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.222.0>", "0x00002aaaac285fc0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_moxi_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.172.0>", "0x00002aaaac285fd8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_reader-@system'},{status,waiting},{initial_call,{proc_li", "y(1) <0.171.0>", "0x00002aaaac285ff0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_archiver-@system'},{status,waiting},{initial_call,{proc_", "y(1) <0.166.0>", "0x00002aaaac286008 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,system_stats_collector},{status,waiting},{initial_call,{proc_li", "y(1) <0.165.0>", "0x00002aaaac286020 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.164.0>", "0x00002aaaac286038 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_worker},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.163.0>", "0x00002aaaac286050 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_stats_event},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.162.0>", "0x00002aaaac286068 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_tick_event},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.161.0>", "0x00002aaaac286080 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_port_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.155.0>", "0x00002aaaac286098 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web_alerts_srv},{status,waiting},{initial_call,{proc_l", "y(1) <0.154.0>", "0x00002aaaac2860b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,hot_keys_keeper},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.153.0>", "0x00002aaaac2860c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.150.0>", "0x00002aaaac2860e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.149.0>", "0x00002aaaac2860f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_doctor},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.147.0>", "0x00002aaaac286110 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_heart},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.145.0>", "0x00002aaaac286128 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.144.0>", "0x00002aaaac286140 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.143.0>", "0x00002aaaac286158 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.142.0>", "0x00002aaaac286170 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_master},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.141.0>", "0x00002aaaac286188 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_rep},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.136.0>", "0x00002aaaac2861a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.129.0>", "0x00002aaaac2861b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_events},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.128.0>", "0x00002aaaac2861d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.127.0>", "0x00002aaaac2861e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log_events},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.126.0>", "0x00002aaaac286200 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.125.0>", "0x00002aaaac286218 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.124.0>", "0x00002aaaac286230 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.119.0>", "0x00002aaaac286248 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.118.0>", "0x00002aaaac286260 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.117.0>", "0x00002aaaac286278 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.92.0>", "0x00002aaaac286290 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.91.0>", "0x00002aaaac2862a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.89.0>", "0x00002aaaac2862c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_server},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.88.0>", "0x00002aaaac2862d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.87.0>", "0x00002aaaac2862f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_late_loader},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.83.0>", "0x00002aaaac286308 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_controller},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.82.0>", "0x00002aaaac286320 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_snmp_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.81.0>", "0x00002aaaac286338 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_checkpoint_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.80.0>", "0x00002aaaac286350 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_tm},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.79.0>", "0x00002aaaac286368 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_recover},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.78.0>", "0x00002aaaac286380 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_locker},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.77.0>", "0x00002aaaac286398 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_subscr},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.76.0>", "0x00002aaaac2863b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_monitor},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.75.0>", "0x00002aaaac2863c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_kernel_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.74.0>", "0x00002aaaac2863e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_event},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.73.0>", "0x00002aaaac2863f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.72.0>", "0x00002aaaac286410 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.71.0>", "0x00002aaaac286428 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.70.0>", "0x00002aaaac286440 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.67.0>", "0x00002aaaac286458 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.66.0>", "0x00002aaaac286470 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.65.0>", "0x00002aaaac286488 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_cluster},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.64.0>", "0x00002aaaac2864a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{net_kernel,ticker,2}},{back", "y(1) <0.63.0>", "0x00002aaaac2864b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,accept_loop,2", "y(1) <0.62.0>", "0x00002aaaac2864d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_kernel},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.61.0>", "0x00002aaaac2864e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,auth},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.60.0>", "0x00002aaaac286500 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_epmd},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.59.0>", "0x00002aaaac286518 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.58.0>", "0x00002aaaac286530 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native},{status,waiting},{initial_call,{inet_getho", "y(1) <0.57.0>", "0x00002aaaac286548 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native_sup},{status,waiting},{initial_call,{proc_l", "y(1) <0.56.0>", "0x00002aaaac286560 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dist_manager},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.55.0>", "0x00002aaaac286578 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_cluster_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.54.0>", "0x00002aaaac286590 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.53.0>", "0x00002aaaac2865a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.52.0>", "0x00002aaaac2865c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,timer_server},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.50.0>", "0x00002aaaac2865d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.49.0>", "0x00002aaaac2865f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.48.0>", "0x00002aaaac286608 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,cpu_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.47.0>", "0x00002aaaac286620 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.46.0>", "0x00002aaaac286638 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,memsup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.45.0>", "0x00002aaaac286650 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disksup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.44.0>", "0x00002aaaac286668 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,os_mon_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.43.0>", "0x00002aaaac286680 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.42.0>", "0x00002aaaac286698 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.41.0>", "0x00002aaaac2866b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,release_handler},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.38.0>", "0x00002aaaac2866c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,overload},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.37.0>", "0x00002aaaac2866e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,alarm_handler},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.36.0>", "0x00002aaaac2866f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_safe_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.35.0>", "0x00002aaaac286710 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.34.0>", "0x00002aaaac286728 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.33.0>", "0x00002aaaac286740 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.32.0>", "0x00002aaaac286758 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_safe_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.27.0>", "0x00002aaaac286770 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.26.0>", "0x00002aaaac286788 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,user},{status,waiting},{initial_call,{user,server,2}},{backtrac", "y(1) <0.25.0>", "0x00002aaaac2867a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.24.0>", "0x00002aaaac2867b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error},{status,waiting},{initial_call,{standard_error,", "y(1) <0.23.0>", "0x00002aaaac2867d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error_sup},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.22.0>", "0x00002aaaac2867e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,code_server},{status,waiting},{initial_call,{erlang,apply,2}},{", "y(1) <0.21.0>", "0x00002aaaac286800 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,file_server_2},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.20.0>", "0x00002aaaac286818 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_group},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.19.0>", "0x00002aaaac286830 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_db},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.17.0>", "0x00002aaaac286848 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.16.0>", "0x00002aaaac286860 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.15.0>", "0x00002aaaac286878 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.14.0>", "0x00002aaaac286890 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_name_server},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.13.0>", "0x00002aaaac2868a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,rex},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backt", "y(1) <0.12.0>", "0x00002aaaac2868c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.11.0>", "0x00002aaaac2868d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.10.0>", "0x00002aaaac2868f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.9.0>", "0x00002aaaac286908 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,application_controller},{status,waiting},{initial_call,{erlang,", "y(1) <0.7.0>", "0x00002aaaac286920 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,error_logger},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.6.0>", "0x00002aaaac286938 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_prim_loader},{status,waiting},{initial_call,{erlang,apply,2", "y(1) <0.3.0>", "0x00002aaaac286950 Return addr 0x00002aaaab4fe278 (diag_handler:do_diag_per_node/0 + 224)", "y(0) [{registered_name,init},{status,waiting},{initial_call,{otp_ring0,start,2}},{back", "y(1) <0.0.0>", "0x00002aaaac286968 Return addr 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 20", "y(0) []","y(1) []", "y(2) [{version,[{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"}", "y(3) [{alerts,[{email,[]},{email_alerts,false},{email_server,[{user,undefined},{pass,'", "y(4) [\"bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz\",\"curl-7.21.1-w64_patched.t", "y(5) [{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"},{inets,\"5", "0x00002aaaac2869a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 208)", "y(1) []","y(2) []","y(3) []", "y(4) <0.12.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,46368}, {total_heap_size,364179}, {links,[]}, {memory,2914120}, {message_queue_len,0}, {reductions,637679}, {trap_exit,false}]}]}, {memory,{2058928128,1594331136,{<10463.462.0>,2172304}}}, {disk, [{"/",13140720,42}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}, {'ns_1@10.1.5.193', [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {manifest, ["bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz", "curl-7.21.1-w64_patched.tar.gz", "ep-engine_1.6.5.3_143_g366c22e-Linux.x86_64.tar.gz", "google-perftools-1.7","google-perftools-1.7.tar.gz", "libconflate_1.6.5_1_g8a42194-Linux.x86_64.tar.gz", "libevent-2.0.7-rc.tar.gz", "libmemcached-0.41_trond-norbye_mingw32-revno895.tar.gz", "libvbucket_1.6.4_1_g5cb588e-Linux.x86_64.tar.gz", "membase-cli_1.6.5_2_gb9c8a7d-Linux.x86_64.tar.gz", "memcached_1.4.4_430_g6235145-Linux.x86_64.tar.gz", "moxi_1.6.5.1_16_g8128746-Linux.x86_64.tar.gz", "ns_server_1.6.5.3_291_gba5fe27.tar.gz", "portsigar_-Linux.x86_64.tar.gz", "sigar_sigar_1.6.4_402_g7809620-Linux.x86_64.tar.gz", "vbucketmigrator_1.7.0r_4_g8428c29-Linux.x86_64.tar.gz", "wallace_1.7.0r-100-g85ee8f5-Linux.x86_64-201104060842"]}, {config, [{alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,'filtered-out'}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up, server_joined,server_left,bucket_created, bucket_deleted,bucket_auth_failed]}]}, {buckets, [{'_vclock',[{'ns_1@10.1.5.192',{11,63469523235}}]}, {configs, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers, ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178']}, {map, [['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}]}]}, {memory_quota,1570}, {nodes_wanted, [{'_vclock',[{'ns_1@10.1.5.192',{2,63469523039}}]}, 'ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193']}, {otp, [{'_vclock',[{'ns_1@10.1.5.193',{1,63469523027}}]}, {cookie,lulxzmudqfmlvwvs}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {rebalance_status, {none, <<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>}}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds, [{'_vclock',[{'ns_1@10.1.5.192',{1,63469522657}}]}, {creds, [{"Administrator",[{password,'filtered-out'}]}]}]}, {{node,'ns_1@10.1.5.178',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.178',membership},inactiveFailed}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.178',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.192',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.192',membership},active}, {{node,'ns_1@10.1.5.192',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.192',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.192',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.192',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.193',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.193',membership},inactiveFailed}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.193',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}]}, {basic_info, [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1167}, {memory_data, {2058928128,1069023232,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}, {processes, [{<10507.0.0>, [{registered_name,init}, {status,waiting}, {initial_call,{otp_ring0,start,2}}, {backtrace, ["Program counter: 0x00002b8796585290 (init:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabfd7230 Return addr 0x0000000000816eb8 ()", "y(0) {state,[{'-root',[<<37 bytes>>]},{'-progname',[<<3 bytes>>]},{'-home',[<<12 bytes"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,84}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<10507.6.0>,<10507.7.0>,<10507.3.0>]}, {memory,108976}, {message_queue_len,0}, {reductions,46711}, {trap_exit,true}]}, {<10507.3.0>, [{registered_name,erl_prim_loader}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b87965e4d20 (erl_prim_loader:loop/3 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabd0a6f0 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) [\"/opt/membase/1.7.0r/bin/ns_server/ebin\",\"/opt/membase/1.7.0r/bin/ns_server/deps", "y(2) <0.2.0>", "y(3) {state,efile,[],none,#Port<0.2>,infinity,undefined,true,{prim_state,false,undefin", "y(4) infinity"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,233}]}, {heap_size,6765}, {total_heap_size,53133}, {links,[#Port<10507.2>,<10507.0.0>]}, {memory,425760}, {message_queue_len,0}, {reductions,959134}, {trap_exit,true}]}, {<10507.6.0>, [{registered_name,error_logger}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666b8a8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_log_mf_h,false,{state,\"/var/membase/logs\",10485760,10,609235,1,{file", "y(3) error_logger","y(4) <0.2.0>", "0x00002b879666b8d8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,46}]}, {heap_size,610}, {total_heap_size,1220}, {links,[<10507.0.0>,<10507.25.0>,#Port<10507.981>]}, {memory,10600}, {message_queue_len,0}, {reductions,212036}, {trap_exit,true}]}, {<10507.7.0>, [{registered_name,application_controller}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796642d50 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) infinity", "y(2) application_controller", "y(3) {state,[],[],[],[{ns_server,<0.52.0>},{mnesia,<0.70.0>},{os_mon,<0.41.0>},{sasl,<", "y(4) application_controller", "y(5) <0.2.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [<10507.41.0>,<10507.52.0>,<10507.70.0>,<10507.9.0>, <10507.32.0>,<10507.0.0>]}, {memory,263664}, {message_queue_len,0}, {reductions,30671}, {trap_exit,true}]}, {<10507.9.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966492f0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.10.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_ser", "y(2) <0.7.0>", "0x00002b8796649310 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10507.7.0>,<10507.10.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<10507.10.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796639790 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) kernel","y(2) <0.11.0>", "y(3) <0.9.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.9.0>,<10507.11.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,72}, {trap_exit,true}]}, {<10507.11.0>, [{registered_name,kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadba358 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_sup},one_for_all,[{child,<0.58.0>,net_sup_dynamic,{erl_distr", "y(4) kernel_sup","y(5) <0.10.0>", "0x00002aaaaadba390 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10507.22.0>,<10507.26.0>,<10507.27.0>,<10507.58.0>, <10507.24.0>,<10507.17.0>,<10507.20.0>,<10507.21.0>, <10507.19.0>,<10507.12.0>,<10507.13.0>,<10507.10.0>]}, {memory,12112}, {message_queue_len,0}, {reductions,1492}, {trap_exit,true}]}, {<10507.12.0>, [{registered_name,rex}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaad1cc9f0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) rpc", "y(3) {1,{<0.2586.0>,{<10509.1166.0>,{#Ref<10509.0.0.23940>,'ns_1@10.1.5.193'}},nil,nil", "y(4) rex","y(5) <0.11.0>", "0x00002aaaad1cca28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,196418}, {total_heap_size,514229}, {links,[<10507.11.0>]}, {memory,4114784}, {message_queue_len,0}, {reductions,2939}, {trap_exit,true}]}, {<10507.13.0>, [{registered_name,global_name_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0f8cc8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) global", "y(3) {state,true,['ns_1@10.1.5.192','ns_1@10.1.5.178'],['ns_1@10.1.5.178','ns_1@10.1.5", "y(4) global_name_server","y(5) <0.11.0>", "0x00002aaaab0f8d00 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,1597}, {total_heap_size,1974}, {links, [<10507.14.0>,<10507.16.0>,<10507.15.0>,<10507.11.0>]}, {memory,16880}, {message_queue_len,0}, {reductions,2067}, {trap_exit,true}]}, {<10507.14.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac82618 (global:loop_the_locker/1 + 1176)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae31268 Return addr 0x00002aaaaac82140 (global:init_the_locker/1 + 384)", "y(0) {multi,[],[],['ns_1@10.1.5.178','ns_1@10.1.5.192'],'ns_1@10.1.5.193',false,false}", "y(1) infinity", "0x00002aaaaae31280 Return addr 0x0000000000816eb8 ()"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10507.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,565}, {trap_exit,true}]}, {<10507.15.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a770 (global:collect_deletions/2 + 152)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb5118 Return addr 0x00002aaaaac8a670 (global:loop_the_deleter/1 + 72)", "y(0) infinity","y(1) []", "y(2) <0.13.0>", "0x00002aaaaadb5138 Return addr 0x0000000000816eb8 ()", "y(0) <0.13.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10507.13.0>]}, {memory,6688}, {message_queue_len,0}, {reductions,231}, {trap_exit,false}]}, {<10507.16.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a9b0 (global:loop_the_registrar/0 + 24)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae53028 Return addr 0x0000000000816eb8 ()", "y(0) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10507.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,248}, {trap_exit,false}]}, {<10507.17.0>, [{registered_name,inet_db}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796675e20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) inet_db", "y(3) {state,inet_db,inet_cache,inet_hosts_byname,inet_hosts_byaddr,inet_hosts_file_byn", "y(4) inet_db","y(5) <0.11.0>", "0x00002b8796675e58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10507.11.0>]}, {memory,26312}, {message_queue_len,0}, {reductions,1457}, {trap_exit,true}]}, {<10507.19.0>, [{registered_name,global_group}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879664a5f8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) global_group", "y(3) {state,no_conf,true,[],[],[],[],[],'nonode@nohost',[],normal,normal}", "y(4) global_group","y(5) <0.11.0>", "0x00002b879664a630 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,84}, {trap_exit,true}]}, {<10507.20.0>, [{registered_name,file_server_2}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab194410 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) file_server","y(3) #Port<0.223>", "y(4) file_server_2","y(5) <0.11.0>", "0x00002aaaab194448 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,362}]}, {heap_size,4181}, {total_heap_size,15127}, {links,[#Port<10507.223>,<10507.11.0>]}, {memory,121816}, {message_queue_len,0}, {reductions,635494}, {trap_exit,true}]}, {<10507.21.0>, [{registered_name,code_server}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaad54b40 (code_server:loop/1 + 128)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbeb980 Return addr 0x0000000000816eb8 ()", "y(0) {state,<0.11.0>,\"/opt/membase/1.7.0r/erlang/lib/erlang\",[\"/opt/membase/1.7.0r/bin", "y(1) <0.11.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,152}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10507.11.0>]}, {memory,284032}, {message_queue_len,0}, {reductions,220561}, {trap_exit,true}]}, {<10507.22.0>, [{registered_name,standard_error_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666c3d0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,standard_error,<0.23.0>,<0.23.0>,{local,standard_error_sup}}", "y(4) standard_error_sup","y(5) <0.11.0>", "0x00002b879666c408 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.11.0>,<10507.23.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10507.23.0>, [{registered_name,standard_error}, {status,waiting}, {initial_call,{standard_error,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad203a0 (standard_error:server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666d2a8 Return addr 0x0000000000816eb8 ()", "y(0) #Port<0.506>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.22.0>,#Port<10507.506>]}, {memory,2664}, {message_queue_len,0}, {reductions,7}, {trap_exit,true}]}, {<10507.24.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb8478 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,user_sup,<0.25.0>,<0.25.0>,{<0.24.0>,user_sup}}", "y(4) <0.24.0>","y(5) <0.11.0>", "0x00002aaaaadb84b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<10507.11.0>,<10507.25.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,71}, {trap_exit,true}]}, {<10507.25.0>, [{registered_name,user}, {status,waiting}, {initial_call,{user,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad765e0 (user:server_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb5c2a8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) {[],[]}", "y(3) #Port<0.523>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,4181}, {total_heap_size,21892}, {links,[<10507.6.0>,<10507.24.0>,#Port<10507.523>]}, {memory,175976}, {message_queue_len,0}, {reductions,48843}, {trap_exit,true}]}, {<10507.26.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666cb20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) kernel_config","y(3) []", "y(4) <0.26.0>","y(5) <0.11.0>", "0x00002b879666cb58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,176}, {trap_exit,true}]}, {<10507.27.0>, [{registered_name,kernel_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae97bb8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_safe_sup},one_for_one,[{child,<0.92.0>,dets,{dets_server,sta", "y(4) kernel_safe_sup","y(5) <0.11.0>", "0x00002aaaaae97bf0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10507.87.0>,<10507.91.0>,<10507.92.0>,<10507.88.0>, <10507.50.0>,<10507.56.0>,<10507.11.0>]}, {memory,7032}, {message_queue_len,0}, {reductions,439}, {trap_exit,true}]}, {<10507.32.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966acad8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.33.0>,{appl_data,sasl,[sasl_sup,alarm_handler,overload,release_handler]", "y(2) <0.7.0>", "0x00002b87966acaf8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10507.7.0>,<10507.33.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,33}, {trap_exit,true}]}, {<10507.33.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796646df0 Return addr 0x0000000000816eb8 ()", "y(0) {state,tty,undefined}","y(1) sasl", "y(2) <0.34.0>","y(3) <0.32.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.32.0>,<10507.34.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,70}, {trap_exit,true}]}, {<10507.34.0>, [{registered_name,sasl_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796647c70 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_sup},one_for_one,[{child,<0.38.0>,release_handler,{release_han", "y(4) sasl_sup","y(5) <0.33.0>", "0x00002b8796647ca8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.35.0>,<10507.38.0>,<10507.33.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,276}, {trap_exit,true}]}, {<10507.35.0>, [{registered_name,sasl_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796633f50 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_safe_sup},one_for_one,[{child,<0.37.0>,overload,{overload,star", "y(4) sasl_safe_sup","y(5) <0.34.0>", "0x00002b8796633f88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.36.0>,<10507.37.0>,<10507.34.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,174}, {trap_exit,true}]}, {<10507.36.0>, [{registered_name,alarm_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666d9c8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,alarm_handler,false,[],false}]", "y(3) alarm_handler","y(4) <0.35.0>", "0x00002b879666d9f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,28}, {trap_exit,true}]}, {<10507.37.0>, [{registered_name,overload}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666e860 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) overload", "y(3) {state,0,0,8.000000e-01,556,1.000000e-01,{0,0},clear}", "y(4) overload","y(5) <0.35.0>", "0x00002b879666e898 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,39}, {trap_exit,false}]}, {<10507.38.0>, [{registered_name,release_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea2440 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) release_handler", "y(3) {state,[],\"/opt/membase/1.7.0r/erlang/lib/erlang\",\"/opt/membase/1.7.0r/erlang/lib", "y(4) release_handler","y(5) <0.34.0>", "0x00002aaaaaea2478 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,4181}, {links,[<10507.34.0>]}, {memory,34208}, {message_queue_len,0}, {reductions,2237}, {trap_exit,false}]}, {<10507.41.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666e128 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.42.0>,{appl_data,os_mon,[os_mon_sup,os_mon_sysinfo,disksup,memsup,cpu_s", "y(2) <0.7.0>", "0x00002b879666e148 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.7.0>,<10507.42.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<10507.42.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796645390 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) os_mon","y(2) <0.43.0>", "y(3) <0.41.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.41.0>,<10507.43.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10507.43.0>, [{registered_name,os_mon_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879664d808 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,os_mon_sup},one_for_one,[{child,<0.47.0>,cpu_sup,{cpu_sup,start_lin", "y(4) os_mon_sup","y(5) <0.42.0>", "0x00002b879664d840 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10507.44.0>,<10507.45.0>,<10507.47.0>,<10507.42.0>]}, {memory,6912}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10507.44.0>, [{registered_name,disksup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966532f0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) disksup", "y(3) {state,80,60000,{unix,linux},[{\"/\",13140720,39},{\"/boot\",101086,13},{\"/dev/shm\",1", "y(4) disksup","y(5) <0.43.0>", "0x00002b8796653328 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,33}]}, {heap_size,2584}, {total_heap_size,5168}, {links,[<10507.43.0>,#Port<10507.879>]}, {memory,42144}, {message_queue_len,0}, {reductions,28365}, {trap_exit,true}]}, {<10507.45.0>, [{registered_name,memsup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966aabe0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) memsup", "y(3) {state,{unix,linux},true,{1069023232,2058928128},{<0.427.0>,1572784},false,60000,", "y(4) memsup","y(5) <0.43.0>", "0x00002b87966aac18 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,48}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<10507.43.0>,<10507.46.0>]}, {memory,26352}, {message_queue_len,0}, {reductions,45853}, {trap_exit,true}]}, {<10507.46.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf49870 (memsup:port_idle/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae3bb20 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.908>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,34}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10507.45.0>,#Port<10507.908>]}, {memory,8592}, {message_queue_len,0}, {reductions,12857}, {trap_exit,true}]}, {<10507.47.0>, [{registered_name,cpu_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadce1f8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) cpu_sup", "y(3) {state,<0.48.0>,{unix,linux}}", "y(4) cpu_sup","y(5) <0.43.0>", "0x00002aaaaadce230 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.43.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,33}, {trap_exit,true}]}, {<10507.48.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3a528 (cpu_sup:measurement_server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcf510 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []","y(5) []", "y(6) []","y(7) []", "y(8) {internal,<0.49.0>,[],{unix,linux}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.49.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,11}, {trap_exit,true}]}, {<10507.49.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3b700 (cpu_sup:port_server_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796638480 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) 6000", "y(2) #Port<0.946>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10507.48.0>,#Port<10507.946>]}, {memory,8592}, {message_queue_len,0}, {reductions,299}, {trap_exit,false}]}, {<10507.50.0>, [{registered_name,timer_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666a588 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) 404","y(2) timer", "y(3) []","y(4) timer_server", "y(5) <0.27.0>", "0x00002b879666a5c0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1891}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10507.460.0>,<10507.480.0>,<10507.2585.0>, <10507.467.0>,<10507.428.0>,<10507.446.0>, <10507.459.0>,<10507.432.0>,<10507.78.0>, <10507.82.0>,<10507.27.0>]}, {memory,9056}, {message_queue_len,0}, {reductions,1272108}, {trap_exit,true}]}, {<10507.52.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02ebd8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.53.0>,{appl_data,ns_server,[ns_server_sup,ns_config,ns_config_sup,ns_co", "y(2) <0.7.0>", "0x00002aaaab02ebf8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.7.0>,<10507.53.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,48}, {trap_exit,true}]}, {<10507.53.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadca470 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) ns_server", "y(2) <0.54.0>","y(3) <0.52.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.52.0>,<10507.54.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,50}, {trap_exit,true}]}, {<10507.54.0>, [{registered_name,ns_server_cluster_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea7508 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_cluster_sup},one_for_one,[{child,<0.427.0>,ns_server_sup,", "y(4) ns_server_cluster_sup","y(5) <0.53.0>", "0x00002aaaaaea7540 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,2584}, {total_heap_size,5168}, {links, [<10507.65.0>,<10507.117.0>,<10507.427.0>, <10507.55.0>,<10507.64.0>,<10507.53.0>]}, {memory,42304}, {message_queue_len,0}, {reductions,1906}, {trap_exit,true}]}, {<10507.55.0>, [{registered_name,dist_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796647520 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dist_manager", "y(3) {state,true,\"10.1.5.193\"}", "y(4) dist_manager","y(5) <0.54.0>", "0x00002b8796647558 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.54.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,912}, {trap_exit,false}]}, {<10507.56.0>, [{registered_name,inet_gethost_native_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcd660 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,inet_gethost_native,<0.57.0>,<0.57.0>,{local,inet_gethost_native_sup}}", "y(4) inet_gethost_native_sup", "y(5) <0.27.0>", "0x00002aaaaadcd698 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.27.0>,<10507.57.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,41}, {trap_exit,true}]}, {<10507.57.0>, [{registered_name,inet_gethost_native}, {status,waiting}, {initial_call,{inet_gethost_native,server_init,2}}, {backtrace, ["Program counter: 0x00002aaaaacc9bd8 (inet_gethost_native:main_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796633598 Return addr 0x0000000000816eb8 ()", "y(0) {state,#Port<0.1012>,8000,8209,12306,<0.56.0>,4,{statistics,0,0,0,0,0,0,0,0}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,297}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10507.56.0>,#Port<10507.1012>]}, {memory,6832}, {message_queue_len,0}, {reductions,12960}, {trap_exit,true}]}, {<10507.58.0>, [{registered_name,net_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796688e28 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,net_sup},one_for_all,[{child,<0.61.0>,net_kernel,{net_kernel,start_", "y(4) net_sup","y(5) <0.11.0>", "0x00002b8796688e60 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links, [<10507.59.0>,<10507.60.0>,<10507.61.0>,<10507.11.0>]}, {memory,8776}, {message_queue_len,0}, {reductions,252}, {trap_exit,true}]}, {<10507.59.0>, [{registered_name,erl_epmd}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796689cc8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) erl_epmd", "y(3) {state,#Port<0.1048>,21100,ns_1}", "y(4) erl_epmd","y(5) <0.58.0>", "0x00002b8796689d00 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.58.0>,#Port<10507.1048>]}, {memory,2664}, {message_queue_len,0}, {reductions,127}, {trap_exit,false}]}, {<10507.60.0>, [{registered_name,auth}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1541d0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) auth", "y(3) {state,lulxzmudqfmlvwvs,16403}", "y(4) auth","y(5) <0.58.0>", "0x00002aaaab154208 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10507.58.0>]}, {memory,6792}, {message_queue_len,0}, {reductions,383}, {trap_exit,true}]}, {<10507.61.0>, [{registered_name,net_kernel}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796658f88 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) net_kernel", "y(3) {state,'ns_1@10.1.5.193','ns_1@10.1.5.193',longnames,{tick,<0.63.0>,15000},7000,s", "y(4) net_kernel","y(5) <0.58.0>", "0x00002b8796658fc0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,18}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10507.63.0>,<10507.422.0>,<10507.500.0>, <10507.58.0>,<10507.62.0>,#Port<10507.1046>]}, {memory,7024}, {message_queue_len,0}, {reductions,2218}, {trap_exit,true}]}, {<10507.62.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,accept_loop,2}}, {backtrace, ["Program counter: 0x00002b87965b7ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc9cf8 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 79","y(1) #Port<0.1046>", "0x00002aaaaadc9d10 Return addr 0x00002aaaaaf84540 (inet_tcp_dist:accept_loop/2 + 96)", "y(0) []", "0x00002aaaaadc9d20 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.1046>", "y(2) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.61.0>]}, {memory,5608}, {message_queue_len,0}, {reductions,525}, {trap_exit,false}]}, {<10507.63.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{net_kernel,ticker,2}}, {backtrace, ["Program counter: 0x00002aaaaad358c8 (net_kernel:ticker_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796638bd8 Return addr 0x0000000000816eb8 ()", "y(0) 15000","y(1) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.61.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,157}, {trap_exit,false}]}, {<10507.64.0>, [{registered_name,ns_cluster}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02b270 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_cluster","y(3) {state}", "y(4) ns_cluster","y(5) <0.54.0>", "0x00002aaaab02b2a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,4181}, {total_heap_size,4181}, {links,[<10507.54.0>]}, {memory,34208}, {message_queue_len,0}, {reductions,1600}, {trap_exit,false}]}, {<10507.65.0>, [{registered_name,mb_mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796639ec0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mb_mnesia_sup},one_for_one,[{child,<0.67.0>,mb_mnesia,{mb_mnesia,st", "y(4) mb_mnesia_sup","y(5) <0.54.0>", "0x00002b8796639ef8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.66.0>,<10507.67.0>,<10507.54.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<10507.66.0>, [{registered_name,mb_mnesia_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb3698 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) mb_mnesia_events","y(4) <0.65.0>", "0x00002aaaaadb36c8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.65.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,32}, {trap_exit,true}]}, {<10507.67.0>, [{registered_name,mb_mnesia}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbfbea8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_mnesia", "y(3) {state,['ns_1@10.1.5.193']}", "y(4) mb_mnesia","y(5) <0.65.0>", "0x00002aaaabbfbee0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<10507.65.0>,<10507.76.0>]}, {memory,67696}, {message_queue_len,0}, {reductions,4697}, {trap_exit,true}]}, {<10507.70.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaeacd40 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.71.0>,{appl_data,mnesia,[mnesia_dumper_load_regulator,mnesia_event,mnes", "y(2) <0.7.0>", "0x00002aaaaaeacd60 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.7.0>,<10507.71.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<10507.71.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcb310 Return addr 0x0000000000816eb8 ()", "y(0) {normal,[]}","y(1) mnesia_sup", "y(2) <0.72.0>","y(3) <0.70.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.70.0>,<10507.72.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10507.72.0>, [{registered_name,mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae98ea8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_sup},one_for_all,[{child,<0.74.0>,mnesia_kernel_sup,{mnesia_", "y(4) mnesia_sup","y(5) <0.71.0>", "0x00002aaaaae98ee0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10507.73.0>,<10507.74.0>,<10507.71.0>]}, {memory,6872}, {message_queue_len,0}, {reductions,207}, {trap_exit,true}]}, {<10507.73.0>, [{registered_name,mnesia_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796631680 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,mnesia_event,false,{state,[],false,[]},false}]", "y(3) mnesia_event","y(4) <0.72.0>", "0x00002b87966316b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10507.72.0>,<10507.76.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,408}, {trap_exit,true}]}, {<10507.74.0>, [{registered_name,mnesia_kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796632990 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_kernel_sup},one_for_all,[{child,<0.83.0>,mnesia_late_loader,", "y(4) mnesia_kernel_sup","y(5) <0.72.0>", "0x00002b87966329c8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10507.77.0>,<10507.81.0>,<10507.82.0>,<10507.83.0>, <10507.79.0>,<10507.80.0>,<10507.78.0>,<10507.75.0>, <10507.76.0>,<10507.72.0>]}, {memory,9016}, {message_queue_len,0}, {reductions,599}, {trap_exit,true}]}, {<10507.75.0>, [{registered_name,mnesia_monitor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcfd368 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_monitor", "y(3) {state,<0.74.0>,[],[],true,[],undefined,[]}", "y(4) mnesia_monitor","y(5) <0.74.0>", "0x00002aaaabcfd3a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,26}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<10507.89.0>,<10507.74.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,6395}, {trap_exit,true}]}, {<10507.76.0>, [{registered_name,mnesia_subscr}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb4530 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_subscr", "y(3) {state,<0.74.0>,20503}", "y(4) mnesia_subscr","y(5) <0.74.0>", "0x00002aaaaadb4568 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.73.0>,<10507.74.0>,<10507.67.0>]}, {memory,2704}, {message_queue_len,0}, {reductions,111}, {trap_exit,true}]}, {<10507.77.0>, [{registered_name,mnesia_locker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab2ac880 (mnesia_locker:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae2a258 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []", "y(5) {state,<0.74.0>}", "0x00002aaaaae2a290 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_locker","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002aaaaae2a2c0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,390}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10507.74.0>]}, {memory,16552}, {message_queue_len,0}, {reductions,201260}, {trap_exit,true}]}, {<10507.78.0>, [{registered_name,mnesia_recover}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0ff538 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_recover", "y(3) {state,<0.74.0>,undefined,undefined,undefined,0,true,[]}", "y(4) mnesia_recover","y(5) <0.74.0>", "0x00002aaaab0ff570 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,12}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10507.74.0>,<10507.50.0>]}, {memory,11712}, {message_queue_len,0}, {reductions,2900}, {trap_exit,true}]}, {<10507.79.0>, [{registered_name,mnesia_tm}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab2bcea0 (mnesia_tm:doit_loop/1 + 216)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796660f10 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,{0,nil},{0,nil},<0.74.0>,[],[],[]}", "y(3) []","y(4) []","y(5) <0.74.0>", "y(6) {0,nil}","y(7) {0,nil}", "0x00002b8796660f58 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_tm","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002b8796660f88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,155}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10507.74.0>]}, {memory,16552}, {message_queue_len,0}, {reductions,462348}, {trap_exit,true}]}, {<10507.80.0>, [{registered_name,mnesia_checkpoint_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796681ea0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_checkpoint_sup},simple_one_for_one,[{child,undefined,mnesia_", "y(4) mnesia_checkpoint_sup","y(5) <0.74.0>", "0x00002b8796681ed8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10507.81.0>, [{registered_name,mnesia_snmp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879665dd60 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_snmp_sup},simple_one_for_one,[{child,undefined,mnesia_snmp_s", "y(4) mnesia_snmp_sup","y(5) <0.74.0>", "0x00002b879665dd98 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10507.82.0>, [{registered_name,mnesia_controller}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb2c80 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_controller", "y(3) {state,<0.74.0>,true,[],[],{0,nil},[],[],{0,nil},undefined,[],[],{interval,#Ref<0", "y(4) mnesia_controller","y(5) <0.74.0>", "0x00002aaaaadb2cb8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10507.74.0>,<10507.50.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,939}, {trap_exit,true}]}, {<10507.83.0>, [{registered_name,mnesia_late_loader}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab2a8d68 (mnesia_late_loader:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966369e8 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,<0.74.0>}", "0x00002b8796636a08 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_late_loader","y(2) []", "y(3) []","y(4) [<0.74.0>]", "0x00002b8796636a38 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,377}, {total_heap_size,377}, {links,[<10507.74.0>]}, {memory,3776}, {message_queue_len,0}, {reductions,172}, {trap_exit,false}]}, {<10507.87.0>, [{registered_name,disk_log_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc95a8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,disk_log_sup},simple_one_for_one,[{child,undefined,disk_log,{disk_l", "y(4) disk_log_sup","y(5) <0.27.0>", "0x00002aaaaadc95e0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<10507.89.0>,<10507.27.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,3508}, {trap_exit,true}]}, {<10507.88.0>, [{registered_name,disk_log_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02dd20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) disk_log_server","y(3) {state,[]}", "y(4) disk_log_server","y(5) <0.27.0>", "0x00002aaaab02dd58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,47}]}, {heap_size,987}, {total_heap_size,1974}, {links,[<10507.89.0>,<10507.27.0>]}, {memory,16672}, {message_queue_len,0}, {reductions,3851}, {trap_exit,true}]}, {<10507.89.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab340f40 (disk_log:loop/1 + 168)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1db10 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) {state,[],[],<0.87.0>,<0.88.0>,217,{arg,latest_log,undefined,\"/var/membase/mnesia", "0x00002aaaaae1db20 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,987}, {total_heap_size,1974}, {links, [<10507.87.0>,<10507.88.0>,<10507.75.0>, #Port<10507.3053>]}, {memory,16672}, {message_queue_len,0}, {reductions,145201}, {trap_exit,true}]}, {<10507.91.0>, [{registered_name,dets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0fa468 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,dets_sup},simple_one_for_one,[{child,undefined,dets,{dets,istart_li", "y(4) dets_sup","y(5) <0.27.0>", "0x00002aaaab0fa4a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10507.27.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,1139}, {trap_exit,true}]}, {<10507.92.0>, [{registered_name,dets}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02e470 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dets_server", "y(3) {state,40996,[<0.27.0>],[]}", "y(4) dets","y(5) <0.27.0>", "0x00002aaaab02e4a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,35}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10507.27.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,1756}, {trap_exit,true}]}, {<10507.117.0>, [{registered_name,ns_config_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae51cd8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_config_sup},rest_for_one,[{child,undefined,ns_config_log,{ns_con", "y(4) ns_config_sup","y(5) <0.54.0>", "0x00002aaaaae51d10 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10507.118.0>,<10507.119.0>,<10507.54.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,580}, {trap_exit,true}]}, {<10507.118.0>, [{registered_name,ns_config_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x0000000000816eb0 (unknown function)", "CP: 0x0000000000816eb8 ()", "arity = 3"," proc_lib"," wake_up", " [gen_event,wake_hib,[<0.117.0>,ns_config_events,[{handler,ns_pubsub,#Ref<0.0.0.7395>,{s"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,774}, {total_heap_size,774}, {links, [<10507.446.0>,<10507.477.0>,<10507.482.0>, <10507.117.0>]}, {memory,7216}, {message_queue_len,0}, {reductions,283914}, {trap_exit,true}]}, {<10507.119.0>, [{registered_name,ns_config}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac3fb868 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config", "y(3) {config,{full,\"/etc/membase/priv/config\",undefined,ns_config_default},[[{director", "y(4) ns_config","y(5) <0.117.0>", "0x00002aaaac3fb8a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,75025}, {total_heap_size,150050}, {links,[<10507.117.0>]}, {memory,1201160}, {message_queue_len,0}, {reductions,97001}, {trap_exit,true}]}, {<10507.422.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_setup,6}}, {backtrace, ["Program counter: 0x00002aaaabe32230 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879665b598 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,8573,4242,3,3}","y(4) normal", "y(5) 'ns_1@10.1.5.193'", "y(6) {net_address,{{10,1,5,192},21100},\"10.1.5.192\",tcp,inet}", "y(7) #Port<0.2534>", "y(8) 'ns_1@10.1.5.192'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10507.61.0>,#Port<10507.2534>]}, {memory,11608}, {message_queue_len,0}, {reductions,1756}, {trap_exit,false}]}, {<10507.427.0>, [{registered_name,ns_server_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb3f868 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_sup},one_for_one,[{child,<0.490.0>,moxi_stats_collector,{", "y(4) ns_server_sup","y(5) <0.54.0>", "0x00002aaaabb3f8a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,75025}, {total_heap_size,196418}, {links, [<10507.462.0>,<10507.476.0>,<10507.481.0>, <10507.482.0>,<10507.490.0>,<10507.479.0>, <10507.480.0>,<10507.477.0>,<10507.474.0>, <10507.475.0>,<10507.468.0>,<10507.430.0>, <10507.459.0>,<10507.460.0>,<10507.446.0>, <10507.428.0>,<10507.429.0>,<10507.54.0>]}, {memory,1572784}, {message_queue_len,0}, {reductions,87800}, {trap_exit,true}]}, {<10507.428.0>, [{registered_name,ns_log}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabe05ee0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) ns_log", "y(3) {state,[{log_entry,{1302,303484,678630},'ns_1@10.1.5.193',ns_node_disco,3,\"Initia", "y(4) ns_log","y(5) <0.427.0>", "0x00002aaaabe05f18 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,4181}, {total_heap_size,10946}, {links,[<10507.50.0>,<10507.427.0>]}, {memory,88368}, {message_queue_len,0}, {reductions,4007}, {trap_exit,false}]}, {<10507.429.0>, [{registered_name,ns_log_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0f98a0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) ns_log_events","y(4) <0.427.0>", "0x00002aaaab0f98d0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10507.427.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,56}, {trap_exit,true}]}, {<10507.430.0>, [{registered_name,ns_node_disco_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879662e738 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_node_disco_sup},rest_for_one,[{child,<0.439.0>,ns_config_rep,{ns", "y(4) ns_node_disco_sup","y(5) <0.427.0>", "0x00002b879662e770 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links, [<10507.432.0>,<10507.439.0>,<10507.431.0>, <10507.427.0>]}, {memory,5760}, {message_queue_len,0}, {reductions,793}, {trap_exit,true}]}, {<10507.431.0>, [{registered_name,ns_node_disco_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x0000000000816eb0 (unknown function)", "CP: 0x0000000000816eb8 ()", "arity = 3"," proc_lib"," wake_up", " [gen_event,wake_hib,[<0.430.0>,ns_node_disco_events,[{handler,menelaus_event,ns_node_di"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,73}, {total_heap_size,73}, {links,[<10507.430.0>]}, {memory,1488}, {message_queue_len,0}, {reductions,128}, {trap_exit,true}]}, {<10507.432.0>, [{registered_name,ns_node_disco}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbd1270 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_node_disco", "y(3) {state,['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193'],{interval,#Ref<0.0", "y(4) ns_node_disco","y(5) <0.430.0>", "0x00002aaaabbd12a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,46}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10507.430.0>,<10507.50.0>]}, {memory,601000}, {message_queue_len,0}, {reductions,4496}, {trap_exit,false}]}, {<10507.439.0>, [{registered_name,ns_config_rep}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac3308e0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config_rep","y(3) {state}", "y(4) ns_config_rep","y(5) <0.430.0>", "0x00002aaaac330918 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,46368}, {total_heap_size,64079}, {links,[<10507.430.0>]}, {memory,513392}, {message_queue_len,0}, {reductions,19118}, {trap_exit,false}]}, {<10507.446.0>, [{registered_name,mb_master}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab69a340 (gen_fsm:loop/7 + 312)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabe1e6e0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_master", "y(3) {state,undefined,'ns_1@10.1.5.192',['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.", "y(4) candidate","y(5) mb_master", "y(6) <0.427.0>", "0x00002aaaabe1e720 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,95}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10507.118.0>,<10507.427.0>,<10507.50.0>]}, {memory,16632}, {message_queue_len,0}, {reductions,42843}, {trap_exit,false}]}, {<10507.459.0>, [{registered_name,ns_heart}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab015c50 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_heart", "y(3) [{meminfo,<<777 bytes>>},{replication,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9c", "y(4) ns_heart","y(5) <0.427.0>", "0x00002aaaab015c88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10507.427.0>,<10507.50.0>]}, {memory,284176}, {message_queue_len,0}, {reductions,1410779}, {trap_exit,false}]}, {<10507.460.0>, [{registered_name,ns_doctor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc76c8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_doctor", "y(3) {state,{dict,3,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{", "y(4) ns_doctor","y(5) <0.427.0>", "0x00002aaaaadc7700 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10507.427.0>,<10507.50.0>]}, {memory,142488}, {message_queue_len,0}, {reductions,58170}, {trap_exit,false}]}, {<10507.462.0>, [{registered_name,menelaus_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb695f8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,menelaus_sup},one_for_one,[{child,<0.467.0>,menelaus_web_alerts_srv", "y(4) menelaus_sup","y(5) <0.427.0>", "0x00002aaaabb69630 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,6765}, {total_heap_size,35422}, {links, [<10507.463.0>,<10507.467.0>,<10507.2585.0>, <10507.427.0>]}, {memory,284256}, {message_queue_len,0}, {reductions,10932}, {trap_exit,true}]}, {<10507.463.0>, [{registered_name,menelaus_web}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab100f78 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mochiweb_socket_server", "y(3) {mochiweb_socket_server,8091,#Fun,{local,menelaus_web}", "y(4) menelaus_web","y(5) <0.462.0>", "0x00002aaaab100fb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10507.486.0>,<10507.1192.0>,<10507.462.0>, <10507.464.0>,#Port<10507.2542>]}, {memory,8816}, {message_queue_len,0}, {reductions,372}, {trap_exit,true}]}, {<10507.464.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab6fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab089478 Return addr 0x00002aaaab6f9b78 (menelaus_web:loop/3 + 26376)", "y(0) {struct,[{buckets,[]}]}", "y(1) {struct,[{buckets,[]}]}", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBuck", "y(3) {mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(4) #Fun", "0x00002aaaab0894a8 Return addr 0x00002aaaab6d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(5) Catch 0x00002aaaab6f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaab0894e0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "0x00002aaaab089508 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,28657}, {total_heap_size,28657}, {links,[<10507.463.0>,#Port<10507.2549>]}, {memory,230200}, {message_queue_len,0}, {reductions,5451}, {trap_exit,false}]}, {<10507.467.0>, [{registered_name,menelaus_web_alerts_srv}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0d3158 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) menelaus_web_alerts_srv", "y(3) {state,[],[],{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],", "y(4) menelaus_web_alerts_srv", "y(5) <0.462.0>", "0x00002aaaab0d3190 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,327}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<10507.50.0>,<10507.462.0>]}, {memory,67696}, {message_queue_len,0}, {reductions,190778}, {trap_exit,false}]}, {<10507.468.0>, [{registered_name,ns_port_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabd48a18 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_port_sup},one_for_one,[{child,<0.492.0>,{memcached,\"./bin/memcac", "y(4) ns_port_sup","y(5) <0.427.0>", "0x00002aaaabd48a50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10507.470.0>,<10507.492.0>,<10507.427.0>]}, {memory,601040}, {message_queue_len,0}, {reductions,3364}, {trap_exit,true}]}, {<10507.470.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796644c20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi,5000,{1302,303828,150702},<0.471.0>}", "y(4) <0.470.0>","y(5) <0.468.0>", "0x00002b8796644c58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10507.468.0>,<10507.471.0>]}, {memory,21472}, {message_queue_len,0}, {reductions,150}, {trap_exit,true}]}, {<10507.471.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab151720 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2543>,moxi,{[\"2011-04-08 16:03:48: (cproxy_config.c.325) env: MOXI", "y(4) <0.471.0>","y(5) <0.470.0>", "0x00002aaaab151758 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10507.470.0>,#Port<10507.2543>]}, {memory,26352}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10507.474.0>, [{registered_name,ns_tick_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1bc10 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7346>,{state,#Fun,ignored},<", "y(3) ns_tick_event","y(4) <0.427.0>", "0x00002aaaaae1bc40 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,25}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10507.479.0>,<10507.427.0>]}, {memory,16592}, {message_queue_len,0}, {reductions,10273}, {trap_exit,true}]}, {<10507.475.0>, [{registered_name,ns_stats_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc07350 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7384>,{state,#Fun,ignored},<", "y(3) ns_stats_event","y(4) <0.427.0>", "0x00002aaaabc07380 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,402}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10507.480.0>,<10507.427.0>]}, {memory,26352}, {message_queue_len,0}, {reductions,26167}, {trap_exit,true}]}, {<10507.476.0>, [{registered_name,ns_bucket_worker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae96160 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) work_queue","y(3) []", "y(4) ns_bucket_worker","y(5) <0.427.0>", "0x00002aaaaae96198 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,1597}, {links,[<10507.427.0>]}, {memory,13536}, {message_queue_len,0}, {reductions,282}, {trap_exit,false}]}, {<10507.477.0>, [{registered_name,ns_bucket_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabca8e18 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_bucket_sup},one_for_one,[{child,<0.816.0>,{per_bucket_sup,\"Failo", "y(4) ns_bucket_sup","y(5) <0.427.0>", "0x00002aaaabca8e50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<10507.427.0>,<10507.816.0>,<10507.118.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,244}, {trap_exit,true}]}, {<10507.479.0>, [{registered_name,system_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796669270 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) system_stats_collector", "y(3) {state,#Port<0.2546>,[{cpu_local_ms,1970828690},{cpu_idle_ms,1810813240},{swap_to", "y(4) system_stats_collector", "y(5) <0.427.0>", "0x00002b87966692a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,144}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10507.427.0>,<10507.474.0>,#Port<10507.2546>]}, {memory,16632}, {message_queue_len,0}, {reductions,47991}, {trap_exit,false}]}, {<10507.480.0>, [{registered_name,'stats_archiver-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796697c08 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"@system\"}", "y(4) 'stats_archiver-@system'", "y(5) <0.427.0>", "0x00002b8796697c40 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,886}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<10507.427.0>,<10507.475.0>,<10507.50.0>]}, {memory,109080}, {message_queue_len,0}, {reductions,4304952}, {trap_exit,false}]}, {<10507.481.0>, [{registered_name,'stats_reader-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966359b8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"@system\"}", "y(4) 'stats_reader-@system'", "y(5) <0.427.0>", "0x00002b87966359f0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.427.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,26}, {trap_exit,false}]}, {<10507.482.0>, [{registered_name,ns_moxi_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabccb798 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_moxi_sup},one_for_one,[{child,<0.483.0>,{\"FailoverNodesTests-4a4", "y(4) ns_moxi_sup","y(5) <0.427.0>", "0x00002aaaabccb7d0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<10507.427.0>,<10507.483.0>,<10507.118.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,2326}, {trap_exit,true}]}, {<10507.483.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796666080 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2547>,moxi,{[\"2011-04-08 16:03:48: (cproxy_config.c.316) env: MOXI", "y(4) <0.483.0>","y(5) <0.482.0>", "0x00002b87966660b8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<10507.482.0>,#Port<10507.2547>]}, {memory,13576}, {message_queue_len,0}, {reductions,168}, {trap_exit,true}]}, {<10507.486.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab6fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab123870 Return addr 0x00002aaaab75fd20 (menelaus_web_buckets:checking_bucket_ac", "y(0) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(1) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsS", "y(3) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) #Fun", "0x00002aaaab1238a0 Return addr 0x00002aaaab6f9a28 (menelaus_web:loop/3 + 26040)", "y(0) #Ref<0.0.0.7487>", "y(1) Catch 0x00002aaaab75fd80 (menelaus_web_buckets:checking_bucket_access/4 + 616)", "y(2) []", "y(3) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) []", "0x00002aaaab1238d0 Return addr 0x00002aaaab6d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(5) Catch 0x00002aaaab6f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaab123908 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "0x00002aaaab123930 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,17711}, {total_heap_size,92736}, {links,[<10507.463.0>,#Port<10507.2550>]}, {memory,742832}, {message_queue_len,0}, {reductions,1305196}, {trap_exit,false}]}, {<10507.490.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabe20d08 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi_stats_collector,5000,{1302,303833,493340},<0.491.0>}", "y(4) <0.490.0>","y(5) <0.427.0>", "0x00002aaaabe20d40 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.427.0>,<10507.491.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<10507.491.0>, [{registered_name,moxi_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0e0488 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) []","y(1) infinity", "y(2) moxi_stats_collector", "y(3) {state,#Port<0.2554>,{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[", "y(4) <0.491.0>","y(5) <0.490.0>", "0x00002aaaab0e04c0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) moxi_stats_collector","y(2) []", "y(3) moxi_stats_collector", "y(4) {local,moxi_stats_collector}", "y(5) <0.490.0>","y(6) <0.490.0>", "0x00002aaaab0e0500 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,15}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [#Port<10507.2636>,<10507.490.0>,#Port<10507.2554>]}, {memory,263544}, {message_queue_len,0}, {reductions,2237829}, {trap_exit,false}]}, {<10507.492.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb1968 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,memcached,5000,{1302,303834,858055},<0.493.0>}", "y(4) <0.492.0>","y(5) <0.468.0>", "0x00002aaaaadb19a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10507.468.0>,<10507.493.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,55}, {trap_exit,true}]}, {<10507.493.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab153600 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2556>,memcached,{[empty],[empty,empty]},undefined,[],0}", "y(4) <0.493.0>","y(5) <0.492.0>", "0x00002aaaab153638 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10507.492.0>,#Port<10507.2556>]}, {memory,8696}, {message_queue_len,0}, {reductions,58}, {trap_exit,true}]}, {<10507.500.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_setup,6}}, {backtrace, ["Program counter: 0x00002aaaabe32230 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0fd648 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,2012,1997,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.193'", "y(6) {net_address,{{10,1,5,178},21100},\"10.1.5.178\",tcp,inet}", "y(7) #Port<0.2561>", "y(8) 'ns_1@10.1.5.178'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10507.61.0>,#Port<10507.2561>]}, {memory,16488}, {message_queue_len,0}, {reductions,1737}, {trap_exit,false}]}, {<10507.816.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b87966007a0 (proc_lib:sync_wait/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb3df8 Return addr 0x00002aaaab78d348 (single_bucket_sup:'-start_link/1-fun-0-", "y(0) infinity","y(1) <0.817.0>", "0x00002aaaaadb3e10 Return addr 0x0000000000816eb8 ()", "y(0) <0.477.0>", "y(1) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\""]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.477.0>,<10507.817.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,30}, {trap_exit,true}]}, {<10507.817.0>, [{registered_name, 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87966007a0 (proc_lib:sync_wait/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae31d90 Return addr 0x00002aaaaac0ffe8 (supervisor:do_start_child/2 + 312)", "y(0) infinity","y(1) <0.818.0>", "0x00002aaaaae31da8 Return addr 0x00002aaaaac0fa28 (supervisor:start_children/3 + 152)", "y(0) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(2) Catch 0x00002aaaaac0ffe8 (supervisor:do_start_child/2 + 312)", "0x00002aaaaae31dc8 Return addr 0x00002aaaaac0f410 (supervisor:init_children/2 + 272)", "y(0) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(1) []", "y(2) [{child,undefined,{ns_vbm_sup,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b9582", "y(3) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "0x00002aaaaae31df0 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) {state,{local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "0x00002aaaaae31e08 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(2) []","y(3) supervisor", "y(4) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(5) <0.816.0>","y(6) <0.816.0>", "0x00002aaaaae31e48 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10507.816.0>,<10507.818.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,664}, {trap_exit,true}]}, {<10507.818.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaaee3ea8 (timer:sleep/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0514e0 Return addr 0x00002aaaab6c2278 (ns_memcached:ensure_bucket/2 + 912)", "y(0) 1000", "0x00002aaaab0514f0 Return addr 0x00002aaaab6be190 (ns_memcached:init/1 + 144)", "y(0) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "y(1) #Port<0.2779>","y(2) []", "0x00002aaaab051510 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) []","y(1) {1302,304035,74311}", "y(2) #Port<0.2779>", "y(3) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "0x00002aaaab051538 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) <0.818.0>","y(2) []", "y(3) ns_memcached","y(4) <0.818.0>", "y(5) <0.817.0>","y(6) <0.817.0>", "0x00002aaaab051578 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,614}]}, {heap_size,17711}, {total_heap_size,46368}, {links,[<10507.817.0>,#Port<10507.2779>]}, {memory,371744}, {message_queue_len,0}, {reductions,1393504}, {trap_exit,false}]}, {<10507.1192.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965b7ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966483b0 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 2986","y(1) #Port<0.2542>", "0x00002b87966483c8 Return addr 0x00002aaaab6dfe88 (mochiweb_socket_server:acceptor_loop/1 ", "y(0) []", "0x00002b87966483d8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) <0.463.0>", "y(2) Catch 0x00002aaaab6dfe88 (mochiweb_socket_server:acceptor_loop/1 + 160)", "0x00002b87966483f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.463.0>]}, {memory,2696}, {message_queue_len,0}, {reductions,38}, {trap_exit,false}]}, {<10507.2585.0>, [{registered_name,hot_keys_keeper}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcaba0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) hot_keys_keeper", "y(3) {state,[],undefined,undefined}", "y(4) hot_keys_keeper","y(5) <0.462.0>", "0x00002aaaaadcabd8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10507.462.0>,<10507.50.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,41}, {trap_exit,false}]}, {<10507.2586.0>, [{registered_name,[]}, {status,running}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b879653c8d8 (unknown function)", "CP: 0x00002aaaab4fe068 (diag_handler:grab_process_info/1 + 64)", "0x00002aaaad751658 Return addr 0x00002aaaab500610 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []", "0x00002aaaad751668 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []","y(1) <0.2586.0>", "0x00002aaaad751680 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,hot_keys_keeper},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.2585.0>", "0x00002aaaad751698 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.1192.0>", "0x00002aaaad7516b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.818.0>", "0x00002aaaad7516c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-5", "y(1) <0.817.0>", "0x00002aaaad7516e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.816.0>", "0x00002aaaad7516f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_setup,6}},", "y(1) <0.500.0>", "0x00002aaaad751710 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.493.0>", "0x00002aaaad751728 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.492.0>", "0x00002aaaad751740 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,moxi_stats_collector},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.491.0>", "0x00002aaaad751758 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.490.0>", "0x00002aaaad751770 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.486.0>", "0x00002aaaad751788 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.483.0>", "0x00002aaaad7517a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_moxi_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.482.0>", "0x00002aaaad7517b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_reader-@system'},{status,waiting},{initial_call,{proc_li", "y(1) <0.481.0>", "0x00002aaaad7517d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_archiver-@system'},{status,waiting},{initial_call,{proc_", "y(1) <0.480.0>", "0x00002aaaad7517e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,system_stats_collector},{status,waiting},{initial_call,{proc_li", "y(1) <0.479.0>", "0x00002aaaad751800 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.477.0>", "0x00002aaaad751818 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_worker},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.476.0>", "0x00002aaaad751830 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_stats_event},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.475.0>", "0x00002aaaad751848 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_tick_event},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.474.0>", "0x00002aaaad751860 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.471.0>", "0x00002aaaad751878 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.470.0>", "0x00002aaaad751890 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_port_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.468.0>", "0x00002aaaad7518a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web_alerts_srv},{status,waiting},{initial_call,{proc_l", "y(1) <0.467.0>", "0x00002aaaad7518c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.464.0>", "0x00002aaaad7518d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.463.0>", "0x00002aaaad7518f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.462.0>", "0x00002aaaad751908 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_doctor},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.460.0>", "0x00002aaaad751920 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_heart},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.459.0>", "0x00002aaaad751938 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_master},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.446.0>", "0x00002aaaad751950 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_rep},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.439.0>", "0x00002aaaad751968 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.432.0>", "0x00002aaaad751980 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_events},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.431.0>", "0x00002aaaad751998 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.430.0>", "0x00002aaaad7519b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log_events},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.429.0>", "0x00002aaaad7519c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.428.0>", "0x00002aaaad7519e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.427.0>", "0x00002aaaad7519f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_setup,6}},", "y(1) <0.422.0>", "0x00002aaaad751a10 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.119.0>", "0x00002aaaad751a28 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.118.0>", "0x00002aaaad751a40 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.117.0>", "0x00002aaaad751a58 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.92.0>", "0x00002aaaad751a70 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.91.0>", "0x00002aaaad751a88 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.89.0>", "0x00002aaaad751aa0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_server},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.88.0>", "0x00002aaaad751ab8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.87.0>", "0x00002aaaad751ad0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_late_loader},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.83.0>", "0x00002aaaad751ae8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_controller},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.82.0>", "0x00002aaaad751b00 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_snmp_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.81.0>", "0x00002aaaad751b18 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_checkpoint_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.80.0>", "0x00002aaaad751b30 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_tm},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.79.0>", "0x00002aaaad751b48 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_recover},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.78.0>", "0x00002aaaad751b60 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_locker},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.77.0>", "0x00002aaaad751b78 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_subscr},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.76.0>", "0x00002aaaad751b90 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_monitor},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.75.0>", "0x00002aaaad751ba8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_kernel_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.74.0>", "0x00002aaaad751bc0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_event},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.73.0>", "0x00002aaaad751bd8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.72.0>", "0x00002aaaad751bf0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.71.0>", "0x00002aaaad751c08 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.70.0>", "0x00002aaaad751c20 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.67.0>", "0x00002aaaad751c38 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.66.0>", "0x00002aaaad751c50 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.65.0>", "0x00002aaaad751c68 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_cluster},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.64.0>", "0x00002aaaad751c80 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{net_kernel,ticker,2}},{back", "y(1) <0.63.0>", "0x00002aaaad751c98 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,accept_loop,2", "y(1) <0.62.0>", "0x00002aaaad751cb0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_kernel},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.61.0>", "0x00002aaaad751cc8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,auth},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.60.0>", "0x00002aaaad751ce0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_epmd},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.59.0>", "0x00002aaaad751cf8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.58.0>", "0x00002aaaad751d10 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native},{status,waiting},{initial_call,{inet_getho", "y(1) <0.57.0>", "0x00002aaaad751d28 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native_sup},{status,waiting},{initial_call,{proc_l", "y(1) <0.56.0>", "0x00002aaaad751d40 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dist_manager},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.55.0>", "0x00002aaaad751d58 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_cluster_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.54.0>", "0x00002aaaad751d70 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.53.0>", "0x00002aaaad751d88 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.52.0>", "0x00002aaaad751da0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,timer_server},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.50.0>", "0x00002aaaad751db8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.49.0>", "0x00002aaaad751dd0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.48.0>", "0x00002aaaad751de8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,cpu_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.47.0>", "0x00002aaaad751e00 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.46.0>", "0x00002aaaad751e18 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,memsup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.45.0>", "0x00002aaaad751e30 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disksup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.44.0>", "0x00002aaaad751e48 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,os_mon_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.43.0>", "0x00002aaaad751e60 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.42.0>", "0x00002aaaad751e78 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.41.0>", "0x00002aaaad751e90 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,release_handler},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.38.0>", "0x00002aaaad751ea8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,overload},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.37.0>", "0x00002aaaad751ec0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,alarm_handler},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.36.0>", "0x00002aaaad751ed8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_safe_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.35.0>", "0x00002aaaad751ef0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.34.0>", "0x00002aaaad751f08 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.33.0>", "0x00002aaaad751f20 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.32.0>", "0x00002aaaad751f38 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_safe_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.27.0>", "0x00002aaaad751f50 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.26.0>", "0x00002aaaad751f68 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,user},{status,waiting},{initial_call,{user,server,2}},{backtrac", "y(1) <0.25.0>", "0x00002aaaad751f80 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.24.0>", "0x00002aaaad751f98 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error},{status,waiting},{initial_call,{standard_error,", "y(1) <0.23.0>", "0x00002aaaad751fb0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error_sup},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.22.0>", "0x00002aaaad751fc8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,code_server},{status,waiting},{initial_call,{erlang,apply,2}},{", "y(1) <0.21.0>", "0x00002aaaad751fe0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,file_server_2},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.20.0>", "0x00002aaaad751ff8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_group},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.19.0>", "0x00002aaaad752010 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_db},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.17.0>", "0x00002aaaad752028 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.16.0>", "0x00002aaaad752040 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.15.0>", "0x00002aaaad752058 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.14.0>", "0x00002aaaad752070 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_name_server},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.13.0>", "0x00002aaaad752088 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,rex},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backt", "y(1) <0.12.0>", "0x00002aaaad7520a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.11.0>", "0x00002aaaad7520b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.10.0>", "0x00002aaaad7520d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.9.0>", "0x00002aaaad7520e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,application_controller},{status,waiting},{initial_call,{erlang,", "y(1) <0.7.0>", "0x00002aaaad752100 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,error_logger},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.6.0>", "0x00002aaaad752118 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_prim_loader},{status,waiting},{initial_call,{erlang,apply,2", "y(1) <0.3.0>", "0x00002aaaad752130 Return addr 0x00002aaaab4fe278 (diag_handler:do_diag_per_node/0 + 224)", "y(0) [{registered_name,init},{status,waiting},{initial_call,{otp_ring0,start,2}},{back", "y(1) <0.0.0>", "0x00002aaaad752148 Return addr 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 20", "y(0) []","y(1) []", "y(2) [{version,[{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"}", "y(3) [{alerts,[{email,[]},{email_alerts,false},{email_server,[{user,undefined},{pass,'", "y(4) [\"bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz\",\"curl-7.21.1-w64_patched.t", "y(5) [{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"},{ns_serve", "0x00002aaaad752180 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 208)", "y(1) []","y(2) []","y(3) []", "y(4) <0.12.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,121393}, {total_heap_size,439204}, {links,[]}, {memory,3514320}, {message_queue_len,0}, {reductions,552273}, {trap_exit,false}]}]}, {memory,{2058928128,1069023232,{<10507.427.0>,1572784}}}, {disk, [{"/",13140720,39}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}] nodes_info = [{struct,[{uptime,<<"1119">>}, {memoryTotal,2075373568}, {memoryFree,309501952}, {mcdMemoryReserved,1583}, {mcdMemoryAllocated,1583}, {otpNode,<<"ns_1@10.1.5.178">>}, {otpCookie,<<"lulxzmudqfmlvwvs">>}, {clusterMembership,<<"inactiveFailed">>}, {status,<<"healthy">>}, {hostname,<<"10.1.5.178:8091">>}, {clusterCompatibility,1}, {version,<<"1.6.5.3_291_gba5fe27">>}, {os,<<"i686-pc-linux-gnu">>}, {ports,{struct,[{proxy,11211},{direct,11210}]}}]}, {struct,[{uptime,<<"1198">>}, {memoryTotal,2058928128}, {memoryFree,464596992}, {mcdMemoryReserved,1570}, {mcdMemoryAllocated,1570}, {otpNode,<<"ns_1@10.1.5.192">>}, {otpCookie,<<"lulxzmudqfmlvwvs">>}, {clusterMembership,<<"active">>}, {status,<<"healthy">>}, {hostname,<<"10.1.5.192:8091">>}, {clusterCompatibility,1}, {version,<<"1.6.5.3_291_gba5fe27">>}, {os,<<"x86_64-unknown-linux-gnu">>}, {ports,{struct,[{proxy,11211},{direct,11210}]}}]}, {struct,[{uptime,<<"1166">>}, {memoryTotal,2058928128}, {memoryFree,989904896}, {mcdMemoryReserved,1570}, {mcdMemoryAllocated,1570}, {otpNode,<<"ns_1@10.1.5.193">>}, {otpCookie,<<"lulxzmudqfmlvwvs">>}, {clusterMembership,<<"inactiveFailed">>}, {status,<<"healthy">>}, {hostname,<<"10.1.5.193:8091">>}, {clusterCompatibility,1}, {version,<<"1.6.5.3_291_gba5fe27">>}, {os,<<"x86_64-unknown-linux-gnu">>}, {ports,{struct,[{proxy,11211},{direct,11210}]}}]}] buckets = [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}] logs: ------------------------------- 2011-04-08 15:59:00.831 ns_node_disco:3:info:cookie update - Initial otp cookie generated: lgirupxdgjtgnqad 2011-04-08 15:59:00.946 menelaus_app:1:info:web start ok - Membase Server has started on web port 8091 on node 'ns_1@10.1.5.178'. 2011-04-08 15:59:00.977 supervisor_cushion:1:warning:port exited too soon after restart - Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s 2011-04-08 15:59:00.978 supervisor_cushion:1:warning:port exited too soon after restart - Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s 2011-04-08 15:59:05.231 ns_port_server:0:info:message - Port server moxi on node 'ns_1@10.1.5.178' exited with status 1. Restarting. Messages: 2011-04-08 15:59:05: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13) 2011-04-08 15:59:05: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8) 2011-04-08 15:59:07: (cproxy.c.329) ERROR: could not listen on port 11211. Please use -Z port_listen=PORT_NUM to specify a different port number. 2011-04-08 15:59:05.231 supervisor_cushion:1:warning:port exited too soon after restart - Service moxi exited on node 'ns_1@10.1.5.178' in 0.01s 2011-04-08 15:59:05.979 supervisor_cushion:1:warning:port exited too soon after restart - Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s 2011-04-08 15:59:05.979 supervisor_cushion:1:warning:port exited too soon after restart - Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s 2011-04-08 16:03:59.919 ns_node_disco:4:info:node up - Node 'ns_1@10.1.5.193' saw that node 'ns_1@10.1.5.178' came up. 2011-04-08 16:04:12.120 menelaus_app:1:info:web start ok - Membase Server has started on web port 8091 on node 'ns_1@10.1.5.178'. 2011-04-08 16:04:12.148 ns_cluster:3:info:message - Node ns_1@10.1.5.178 joined cluster 2011-04-08 16:04:14.383 ns_orchestrator:4:info:message - Starting rebalance, KeepNodes = ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178'], EjectNodes = [] 2011-04-08 16:04:14.666 ns_port_server:0:info:message - Port server memcached on node 'ns_1@10.1.5.178' exited with status 137. Restarting. Messages: 2011-04-08 16:04:14.666 supervisor_cushion:1:warning:port exited too soon after restart - Service memcached exited on node 'ns_1@10.1.5.178' in 2.53s 2011-04-08 16:04:16.534 ns_memcached:1:info:message - Bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" loaded on node 'ns_1@10.1.5.193' in 2 seconds. 2011-04-08 16:04:26.688 ns_memcached:1:info:message - Bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" loaded on node 'ns_1@10.1.5.178' in 1 seconds. 2011-04-08 16:05:00.525 ns_orchestrator:1:info:message - Rebalance completed successfully. 2011-04-08 16:06:59.506 ns_memcached:2:info:message - Shutting down bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" on 'ns_1@10.1.5.193' for server shutdown 2011-04-08 16:06:59.870 ns_orchestrator:6:info:message - Failed over 'ns_1@10.1.5.193': ok 2011-04-08 16:06:59.960 ns_rebalancer:1:info:message - Data has been lost for 33% of vbuckets in bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561". 2011-04-08 16:07:00.085 ns_orchestrator:6:info:message - Failed over 'ns_1@10.1.5.178': ok 2011-04-08 16:07:10.366 ns_memcached:2:info:message - Shutting down bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" on 'ns_1@10.1.5.178' for server shutdown 2011-04-08 16:07:15.094 ns_orchestrator:4:info:message - Starting rebalance, KeepNodes = ['ns_1@10.1.5.192'], EjectNodes = ['ns_1@10.1.5.193', 'ns_1@10.1.5.178'] 2011-04-08 16:07:26.115 ns_orchestrator:2:info:message - Rebalance exited with reason {wait_for_memcached_failed, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", ['ns_1@10.1.5.193','ns_1@10.1.5.178']} logs_node: ------------------------------- INFO REPORT <3.54.0> 2011-04-08 15:59:00 =============================================================================== nonode@nohost:<3.54.0>:log_os_info:25: OS type: {unix,linux} Version: {2,6,18} Runtime info: [{otp_release,"R13B03"}, {erl_version,"5.7.4"}, {erl_version_long, "Erlang R13B03 (erts-5.7.4) [source] [rq:1] [async-threads:16] [kernel-poll:false]\n"}, {system_arch_raw,"i686-pc-linux-gnu"}, {system_arch,"i686-pc-linux-gnu"}, {localtime,{{2011,4,8},{15,59,0}}}, {memory, [{total,7692512}, {processes,648628}, {processes_used,642068}, {system,7043884}, {atom,254677}, {atom_used,253442}, {binary,44768}, {code,1775473}, {ets,153412}]}, {loaded, [ns_info,log_os_info,misc,ns_log_mf_h, ns_server_cluster_sup,ns_server,timer,io_lib_fread, cpu_sup,memsup,disksup,os_mon,io_lib_pretty,unicode, io_lib_format,io_lib,io,sasl_report,release_handler, calendar,overload,alarm_handler,sasl_report_tty_h,sasl, ns_bootstrap,file_io_server,orddict,erl_eval,file,c, error_logger_tty_h,queue,kernel_config,user,user_sup, supervisor_bridge,standard_error,gb_sets, hipe_unified_loader,packages,code_server,code,file_server, net_kernel,global_group,erl_distribution,string,ets, filename,inet_gethost_native,inet_parse,inet,inet_udp,os, inet_config,inet_db,global,gb_trees,rpc,dict,supervisor, kernel,application_master,sys,application,gen_server, erl_parse,proplists,erl_scan,lists,application_controller, proc_lib,gen,gen_event,error_logger,heart,error_handler, erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {applications, [{kernel,"ERTS CXC 138 10","2.13.4"}, {stdlib,"ERTS CXC 138 10","1.16.4"}, {os_mon,"CPO CXC 138 46","2.2.4"}, {ns_server,"Membase server","1.6.5.3_291_gba5fe27"}, {menelaus,"Membase menelaus","1.6.5.3_291_gba5fe27"}, {sasl,"SASL CXC 138 11","2.1.8"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,44}, {node,nonode@nohost}, {nodes,[]}, {registered, [overload,init,erl_prim_loader,alarm_handler,error_logger, rex,user,kernel_sup,global_name_server,sasl_sup,inet_db, os_mon_sup,code_server,file_server_2,kernel_safe_sup, cpu_sup,memsup,application_controller,sasl_safe_sup, disksup,standard_error,ns_server_cluster_sup, standard_error_sup,timer_server,global_group, release_handler]}, {cookie,nocookie}, {wordsize,4}, {wall_clock,0}] INFO REPORT <3.55.0> 2011-04-08 15:59:00 =============================================================================== reading ip config from "/etc/membase/priv/ip" PROGRESS REPORT <3.56.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,inet_gethost_native_sup} started [{pid,<3.57.0>},{mfa,{inet_gethost_native,init,[[]]}}] PROGRESS REPORT <3.27.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<3.56.0>}, {name,inet_gethost_native_sup}, {mfa,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] INFO REPORT <3.55.0> 2011-04-08 15:59:00 =============================================================================== nonode@nohost:<3.55.0>:dist_manager:109: Attempting to bring up net_kernel with name 'ns_1@10.1.5.178' PROGRESS REPORT <3.58.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,net_sup} started [{pid,<3.59.0>}, {name,erl_epmd}, {mfa,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <3.58.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,net_sup} started [{pid,<3.60.0>}, {name,auth}, {mfa,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.58.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,net_sup} started [{pid,<0.61.0>}, {name,net_kernel}, {mfa,{net_kernel,start_link,[['ns_1@10.1.5.178',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.11.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,kernel_sup} started [{pid,<0.58.0>}, {name,net_sup_dynamic}, {mfa,{erl_distribution,start_link,[['ns_1@10.1.5.178',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <0.54.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.55.0>}, {name,dist_manager}, {mfa,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.64.0>}, {name,ns_cluster}, {mfa,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.65.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<0.66.0>}, {name,mb_mnesia_events}, {mfa,{gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.72.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<0.73.0>}, {name,mnesia_event}, {mfa,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.75.0>}, {name,mnesia_monitor}, {mfa,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.76.0>}, {name,mnesia_subscr}, {mfa,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.77.0>}, {name,mnesia_locker}, {mfa,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.78.0>}, {name,mnesia_recover}, {mfa,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.79.0>}, {name,mnesia_tm}, {mfa,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.80.0>}, {name,mnesia_checkpoint_sup}, {mfa,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.81.0>}, {name,mnesia_snmp_sup}, {mfa,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.82.0>}, {name,mnesia_controller}, {mfa,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.83.0>}, {name,mnesia_late_loader}, {mfa,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.72.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<0.74.0>}, {name,mnesia_kernel_sup}, {mfa,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.7.0> 2011-04-08 15:59:00 =============================================================================== application mnesia started_at 'ns_1@10.1.5.178' PROGRESS REPORT <0.27.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.87.0>}, {name,disk_log_sup}, {mfa,{disk_log_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <0.27.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.88.0>}, {name,disk_log_server}, {mfa,{disk_log_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.27.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.91.0>}, {name,dets_sup}, {mfa,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <0.27.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.92.0>}, {name,dets}, {mfa,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:411: Committed schema to disk. INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:247: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@10.1.5.178']}, {debug,verbose}, {directory,"/var/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,[local_config,cluster,schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@10.1.5.178']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<0.73.0>,<0.67.0>]}, {tables,[local_config,cluster,schema]}, {transaction_commits,5}, {transaction_failures,0}, {transaction_log_writes,3}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {version,"4.4.12"}] Peers: ['ns_1@10.1.5.178'] INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,schema, [{name,schema}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,false}, {record_name,schema}, {attributes,[table,cstruct]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303540,484628},'ns_1@10.1.5.178'}}, {version,{{3,0},{'ns_1@10.1.5.178',{1302,303540,508120}}}}]}, {tid,3,<0.86.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:194: Peers: ['ns_1@10.1.5.178'] INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:194: Peers: ['ns_1@10.1.5.178'] INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303540,797544},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,5,<0.111.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303540,797544},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,5,<0.111.0>}} PROGRESS REPORT <0.65.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<0.67.0>}, {name,mb_mnesia}, {mfa,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.65.0>}, {name,mb_mnesia_sup}, {mfa,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.117.0> 2011-04-08 15:59:00 =============================================================================== loading config from "/etc/membase/priv/config" PROGRESS REPORT <0.117.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.118.0>}, {name,ns_config_events}, {mfa,{gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.117.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.119.0>}, {name,ns_config}, {mfa,{ns_config,start_link, ["/etc/membase/priv/config",ns_config_default]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.117.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.121.0>}, {name,ns_config_isasl_sync}, {mfa,{ns_config_isasl_sync,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== isasl_sync init: ["/var/membase/data/isasl.pw","_admin","_admin"] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== isasl_sync init buckets: [] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== Writing isasl passwd file: "/var/membase/data/isasl.pw" PROGRESS REPORT <0.117.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.122.0>}, {name,ns_config_log}, {mfa,{ns_config_log,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.117.0>}, {name,ns_config_sup}, {mfa,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.125.0> 2011-04-08 15:59:00 =============================================================================== ns_log:init(): Couldn't load logs from "/var/membase/data/ns_log": {error, enoent} PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.125.0>}, {name,ns_log}, {mfa,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.126.0>}, {name,ns_log_events}, {mfa,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.127.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.128.0>}, {name,ns_node_disco_events}, {mfa,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.129.0> 2011-04-08 15:59:00 =============================================================================== Initting ns_node_disco with [] INFO REPORT <0.130.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.130.0> 2011-04-08 15:59:00 =============================================================================== ns_log: logging ns_node_disco:3:Initial otp cookie generated: lgirupxdgjtgnqad INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: otp -> [{cookie,lgirupxdgjtgnqad}] INFO REPORT <0.130.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178'], with cookie: lgirupxdgjtgnqad INFO REPORT <0.130.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178'], with cookie: lgirupxdgjtgnqad PROGRESS REPORT <0.127.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.129.0>}, {name,ns_node_disco}, {mfa,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.127.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.134.0>}, {name,ns_node_disco_log}, {mfa,{ns_node_disco_log,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.127.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.135.0>}, {name,ns_node_disco_conf_events}, {mfa,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.136.0> 2011-04-08 15:59:00 =============================================================================== ns_config_rep init pulling INFO REPORT <0.136.0> 2011-04-08 15:59:00 =============================================================================== ns_config_rep init pushing INFO REPORT <0.136.0> 2011-04-08 15:59:00 =============================================================================== ns_config_rep init reannouncing INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: otp -> [{cookie,lgirupxdgjtgnqad}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: alerts -> [{email,[]}, {email_alerts,false}, [{user,undefined}, {pass,"********"}, {addr,undefined}, {port,undefined}, {encrypt,false}], {alerts,[server_down,server_unresponsive,server_up,server_joined,server_left, bucket_created,bucket_deleted,bucket_auth_failed]}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: memory_quota -> 1583 INFO REPORT <0.138.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.178'] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: port_servers -> [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary","-r", "-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}] INFO REPORT <0.139.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.138.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178'], with cookie: lgirupxdgjtgnqad INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> active INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.139.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178'], with cookie: lgirupxdgjtgnqad INFO REPORT <0.138.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178'], with cookie: lgirupxdgjtgnqad INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.139.0> 2011-04-08 15:59:00 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178'], with cookie: lgirupxdgjtgnqad INFO REPORT <0.136.0> 2011-04-08 15:59:00 =============================================================================== Pushing config PROGRESS REPORT <0.127.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.136.0>}, {name,ns_config_rep}, {mfa,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.127.0>}, {name,ns_node_disco_sup}, {mfa,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.136.0> 2011-04-08 15:59:00 =============================================================================== Pushing config done INFO REPORT <0.141.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.141.0>:mb_master:83: I'm the only node, so I'm the master. INFO REPORT <0.142.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.142.0>:misc:725: start_singleton(gen_fsm, ns_orchestrator, [], []): started as <0.143.0> on 'ns_1@10.1.5.178' PROGRESS REPORT <0.142.0> 2011-04-08 15:59:00 =============================================================================== supervisor {<0.142.0>,mb_master_sup} started [{pid,<0.143.0>}, {name,ns_orchestrator}, {mfa,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] INFO REPORT <0.142.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.142.0>:misc:725: start_singleton(gen_server, ns_tick, [], []): started as <0.144.0> on 'ns_1@10.1.5.178' PROGRESS REPORT <0.142.0> 2011-04-08 15:59:00 =============================================================================== supervisor {<0.142.0>,mb_master_sup} started [{pid,<0.144.0>}, {name,ns_tick}, {mfa,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.141.0>}, {name,mb_master}, {mfa,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.145.0>}, {name,ns_heart}, {mfa,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.147.0>}, {name,ns_doctor}, {mfa,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.147.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.147.0>:ns_doctor:82: Got initial status [{'ns_1@10.1.5.178', [{last_heard, {1302,303540, 884473}}, {active_buckets, []}, {memory, [{total, 10268848}, {processes, 1271592}, {processes_used, 1268200}, {system, 8997256}, {atom,428705}, {atom_used, 414129}, {binary,27232}, {code,3439849}, {ets,244916}]}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.13.4"}, {mnesia, "4.4.12"}, {stdlib, "1.16.4"}, {os_mon, "2.2.4"}, {ns_server, "1.6.5.3_291_gba5fe27"}, {menelaus, "1.6.5.3_291_gba5fe27"}, {sasl, "2.1.8"}]}, {system_arch, "i686-pc-linux-gnu"}, {wall_clock,0}, {memory_data, {2075373568, 1701920768, {<0.7.0>, 71344}}}, {disk_data, [{"/",11173104, 41}, {"/boot", 101086,13}, {"/dev/shm", 1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 362468 kB\nBuffers: 246308 kB\nCached: 1246912 kB\nSwapCached: 0 kB\nActive: 676900 kB\nInactive: 888540 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 360832 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 172892 kB\nWriteback: 0 kB\nAnonPages: 72184 kB\nMapped: 26632 kB\nSlab: 86568 kB\nPageTables: 1972 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 206812 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory, 2075373568}, {free_swap, 4160741376}, {total_swap, 4160741376}, {cached_memory, 1276837888}, {buffered_memory, 252219392}, {free_memory, 371167232}, {total_memory, 2075373568}]}, {statistics, [{wall_clock, {728,486}}, {context_switches, {7570,0}}, {garbage_collection, {1511, 2722477,0}}, {io, {{input, 3607175}, {output, 185943}}}, {reductions, {1251604, 1251604}}, {run_queue,0}, {runtime, {330, 330}}]}]}] PROGRESS REPORT <0.149.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.150.0>}, {name,menelaus_web}, {mfa,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.149.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.152.0>}, {name,menelaus_event}, {mfa,{menelaus_event,start_link,[]}}, {restart_type,transient}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.149.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.153.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.149.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.154.0>}, {name,menelaus_web_alerts_srv}, {mfa,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== ns_log: logging menelaus_app:1:Membase Server has started on web port 8091 on node 'ns_1@10.1.5.178'. PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.149.0>}, {name,menelaus}, {mfa,{menelaus_app,start_subapp,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.155.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.156.0>}, {name,ns_port_init}, {mfa,{ns_port_init,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.157.0> 2011-04-08 15:59:00 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.155.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.157.0>}, {name, {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.159.0> 2011-04-08 15:59:00 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.155.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.159.0>}, {name, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p","11210", "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.155.0>}, {name,ns_port_sup}, {mfa,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.161.0>}, {name,ns_tick_event}, {mfa,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.162.0>}, {name,ns_stats_event}, {mfa,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.163.0>}, {name,ns_bucket_worker}, {mfa,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.164.0>}, {name,ns_bucket_sup}, {mfa,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.165.0>}, {name,system_stats_collector}, {mfa,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.166.0>}, {name,{stats_archiver,"@system"}}, {mfa,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 15:59:00 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-minute' PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.169.0>}, {name,{stats_reader,"@system"}}, {mfa,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.172.0>}, {name,ns_moxi_sup}, {mfa,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.173.0> 2011-04-08 15:59:00 =============================================================================== starting moxi_stats_collector with delay of 5000 PROGRESS REPORT <0.124.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.173.0>}, {name,moxi_stats_collector}, {mfa,{supervisor_cushion,start_link, [moxi_stats_collector,5000, moxi_stats_collector,start_link,[]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:59:00 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.124.0>}, {name,ns_server_sup}, {mfa,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.7.0> 2011-04-08 15:59:00 =============================================================================== application ns_server started_at 'ns_1@10.1.5.178' CRASH REPORT <0.174.0> 2011-04-08 15:59:00 =============================================================================== Crashing process initial_call {moxi_stats_collector,init,['Argument__1']} pid <0.174.0> registered_name [] error_info {exit,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,init_it,6},{proc_lib,init_p_do_apply,3}]} ancestors [<0.173.0>,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [] links [<0.173.0>] dictionary [] trap_exit false status running heap_size 6765 stack_size 24 reductions 339 INFO REPORT <0.173.0> 2011-04-08 15:59:00 =============================================================================== Cushion got unexpected info supervising moxi_stats_collector: {ack,<0.174.0>, {error, {{badmatch, {error, econnrefused}}, [{moxi_stats_collector, connect_moxi, 1}, {moxi_stats_collector, init,1}, {gen_server, init_it,6}, {proc_lib, init_p_do_apply, 3}]}}} INFO REPORT <0.173.0> 2011-04-08 15:59:00 =============================================================================== ns_log: logging supervisor_cushion:1:Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s INFO REPORT <0.173.0> 2011-04-08 15:59:00 =============================================================================== Cushion managed supervisor for moxi_stats_collector failed: {{badmatch, {error, econnrefused}}, [{moxi_stats_collector, connect_moxi, 1}, {moxi_stats_collector, init,1}, {gen_server, init_it,6}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <0.173.0> 2011-04-08 15:59:00 =============================================================================== ns_log: logging supervisor_cushion:1:Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303540,974236},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,6,<0.168.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303540,974236},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,6,<0.168.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-hour' INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,7114},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,7,<0.180.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,7114},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,7,<0.180.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-day' INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,53938},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,8,<0.187.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,53938},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,8,<0.187.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-week' INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,64213},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,9,<0.194.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,64213},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,9,<0.194.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-month' INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,69114},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,10,<0.201.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,69114},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,10,<0.201.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-year' INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,98932},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,11,<0.208.0>}} INFO REPORT <0.67.0> 2011-04-08 15:59:01 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303541,98932},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,11,<0.208.0>}} INFO REPORT <0.118.0> 2011-04-08 15:59:05 =============================================================================== unsupervising port: {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,stderr_to_stdout,stream]} INFO REPORT <0.118.0> 2011-04-08 15:59:05 =============================================================================== supervising port: {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]} INFO REPORT <0.216.0> 2011-04-08 15:59:05 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.155.0> 2011-04-08 15:59:05 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.216.0>}, {name, {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.118.0> 2011-04-08 15:59:05 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 15:59:05 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.136.0> 2011-04-08 15:59:05 =============================================================================== Pushing config INFO REPORT <0.136.0> 2011-04-08 15:59:05 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 15:59:05 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.118.0> 2011-04-08 15:59:05 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.136.0> 2011-04-08 15:59:05 =============================================================================== Pushing config INFO REPORT <0.136.0> 2011-04-08 15:59:05 =============================================================================== Pushing config done INFO REPORT <0.151.0> 2011-04-08 15:59:05 =============================================================================== menelaus_web streaming socket closed by client INFO REPORT <0.217.0> 2011-04-08 15:59:05 =============================================================================== ns_log: logging ns_port_server:0:Port server moxi on node 'ns_1@10.1.5.178' exited with status 1. Restarting. Messages: 2011-04-08 15:59:05: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13) 2011-04-08 15:59:05: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8) 2011-04-08 15:59:07: (cproxy.c.329) ERROR: could not listen on port 11211. Please use -Z port_listen=PORT_NUM to specify a different port number. INFO REPORT <0.217.0> 2011-04-08 15:59:05 =============================================================================== moxi<0.217.0>: 2011-04-08 15:59:05: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13) moxi<0.217.0>: 2011-04-08 15:59:05: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8) moxi<0.217.0>: 2011-04-08 15:59:07: (cproxy.c.329) ERROR: could not listen on port 11211. Please use -Z port_listen=PORT_NUM to specify a different port number. ERROR REPORT <0.217.0> 2011-04-08 15:59:05 =============================================================================== ** Generic server <0.217.0> terminating ** Last message in was {#Port<0.2192>,{exit_status,1}} ** When Server state == {state,#Port<0.2192>,moxi, {["2011-04-08 15:59:07: (cproxy.c.329) ERROR: could not listen on port 11211. Please use -Z port_listen=PORT_NUM to specify a different port number.", "2011-04-08 15:59:05: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8)"], ["2011-04-08 15:59:05: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13)"]}, {ok,{1302303546229990,#Ref<0.0.0.977>}}, ["2011-04-08 15:59:07: (cproxy.c.329) ERROR: could not listen on port 11211. Please use -Z port_listen=PORT_NUM to specify a different port number.", "2011-04-08 15:59:05: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8)", "2011-04-08 15:59:05: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13)"], 0} ** Reason for termination == ** {abnormal,1} CRASH REPORT <0.217.0> 2011-04-08 15:59:05 =============================================================================== Crashing process initial_call {ns_port_server,init,['Argument__1']} pid <0.217.0> registered_name [] error_info {exit,{abnormal,1}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [<0.216.0>,ns_port_sup,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [{'EXIT',#Port<0.2192>,normal}] links [<0.216.0>] dictionary [] trap_exit true status running heap_size 1597 stack_size 24 reductions 1574 INFO REPORT <0.216.0> 2011-04-08 15:59:05 =============================================================================== Cushion managed supervisor for moxi failed: {abnormal,1} INFO REPORT <0.216.0> 2011-04-08 15:59:05 =============================================================================== ns_log: logging supervisor_cushion:1:Service moxi exited on node 'ns_1@10.1.5.178' in 0.01s ERROR REPORT <0.173.0> 2011-04-08 15:59:05 =============================================================================== ** Generic server <0.173.0> terminating ** Last message in was {die, {error,cushioned_supervisor, {ack,<0.174.0>, {error, {{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}}} ** When Server state == {state,moxi_stats_collector,5000, {1302,303540,977130}, undefined} ** Reason for termination == ** {error,cushioned_supervisor, {ack,<0.174.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}} CRASH REPORT <0.173.0> 2011-04-08 15:59:05 =============================================================================== Crashing process initial_call {supervisor_cushion,init,['Argument__1']} pid <0.173.0> registered_name [] error_info {exit,{error,cushioned_supervisor, {ack,<0.174.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [{die,{error,cushioned_supervisor, {{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}] links [<0.124.0>] dictionary [] trap_exit true status running heap_size 610 stack_size 24 reductions 1096 SUPERVISOR REPORT <0.124.0> 2011-04-08 15:59:05 =============================================================================== Reporting supervisor {local,ns_server_sup} Child process errorContext child_terminated reason {error,cushioned_supervisor, {ack,<0.174.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}} pid <0.173.0> name moxi_stats_collector start_function {supervisor_cushion,start_link, [moxi_stats_collector,5000,moxi_stats_collector, start_link,[]]} restart_type permanent shutdown 10 child_type worker INFO REPORT <0.225.0> 2011-04-08 15:59:05 =============================================================================== starting moxi_stats_collector with delay of 5000 PROGRESS REPORT <0.124.0> 2011-04-08 15:59:05 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.225.0>}, {name,moxi_stats_collector}, {mfa,{supervisor_cushion,start_link, [moxi_stats_collector,5000, moxi_stats_collector,start_link,[]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] CRASH REPORT <0.226.0> 2011-04-08 15:59:05 =============================================================================== Crashing process initial_call {moxi_stats_collector,init,['Argument__1']} pid <0.226.0> registered_name [] error_info {exit,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,init_it,6},{proc_lib,init_p_do_apply,3}]} ancestors [<0.225.0>,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [] links [<0.225.0>] dictionary [] trap_exit false status running heap_size 6765 stack_size 24 reductions 339 INFO REPORT <0.225.0> 2011-04-08 15:59:05 =============================================================================== Cushion got unexpected info supervising moxi_stats_collector: {ack,<0.226.0>, {error, {{badmatch, {error, econnrefused}}, [{moxi_stats_collector, connect_moxi, 1}, {moxi_stats_collector, init,1}, {gen_server, init_it,6}, {proc_lib, init_p_do_apply, 3}]}}} INFO REPORT <0.225.0> 2011-04-08 15:59:05 =============================================================================== ns_log: logging supervisor_cushion:1:Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s INFO REPORT <0.225.0> 2011-04-08 15:59:05 =============================================================================== Cushion managed supervisor for moxi_stats_collector failed: {{badmatch, {error, econnrefused}}, [{moxi_stats_collector, connect_moxi, 1}, {moxi_stats_collector, init,1}, {gen_server, init_it,6}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <0.225.0> 2011-04-08 15:59:05 =============================================================================== ns_log: logging supervisor_cushion:1:Service moxi_stats_collector exited on node 'ns_1@10.1.5.178' in 0.00s ERROR REPORT <0.216.0> 2011-04-08 15:59:10 =============================================================================== ** Generic server <0.216.0> terminating ** Last message in was {die,{error,cushioned_supervisor,{abnormal,1}}} ** When Server state == {state,moxi,5000,{1302,303545,226189},undefined} ** Reason for termination == ** {error,cushioned_supervisor,{abnormal,1}} CRASH REPORT <0.216.0> 2011-04-08 15:59:10 =============================================================================== Crashing process initial_call {supervisor_cushion,init,['Argument__1']} pid <0.216.0> registered_name [] error_info {exit,{error,cushioned_supervisor,{abnormal,1}}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_port_sup,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [] links [<0.155.0>] dictionary [] trap_exit true status running heap_size 1597 stack_size 24 reductions 623 SUPERVISOR REPORT <0.155.0> 2011-04-08 15:59:10 =============================================================================== Reporting supervisor {local,ns_port_sup} Child process errorContext child_terminated reason {error,cushioned_supervisor,{abnormal,1}} pid <0.216.0> name {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]} start_function {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]]]} restart_type permanent shutdown 10 child_type worker INFO REPORT <0.227.0> 2011-04-08 15:59:10 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.155.0> 2011-04-08 15:59:10 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.227.0>}, {name, {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] ERROR REPORT <0.225.0> 2011-04-08 15:59:10 =============================================================================== ** Generic server <0.225.0> terminating ** Last message in was {die, {error,cushioned_supervisor, {ack,<0.226.0>, {error, {{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}}} ** When Server state == {state,moxi_stats_collector,5000, {1302,303545,978368}, undefined} ** Reason for termination == ** {error,cushioned_supervisor, {ack,<0.226.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}} CRASH REPORT <0.225.0> 2011-04-08 15:59:10 =============================================================================== Crashing process initial_call {supervisor_cushion,init,['Argument__1']} pid <0.225.0> registered_name [] error_info {exit,{error,cushioned_supervisor, {ack,<0.226.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [{die,{error,cushioned_supervisor, {{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}] links [<0.124.0>] dictionary [] trap_exit true status running heap_size 610 stack_size 24 reductions 1088 SUPERVISOR REPORT <0.124.0> 2011-04-08 15:59:10 =============================================================================== Reporting supervisor {local,ns_server_sup} Child process errorContext child_terminated reason {error,cushioned_supervisor, {ack,<0.226.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}} pid <0.225.0> name moxi_stats_collector start_function {supervisor_cushion,start_link, [moxi_stats_collector,5000,moxi_stats_collector, start_link,[]]} restart_type permanent shutdown 10 child_type worker INFO REPORT <0.233.0> 2011-04-08 15:59:10 =============================================================================== starting moxi_stats_collector with delay of 5000 PROGRESS REPORT <0.124.0> 2011-04-08 15:59:10 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.233.0>}, {name,moxi_stats_collector}, {mfa,{supervisor_cushion,start_link, [moxi_stats_collector,5000, moxi_stats_collector,start_link,[]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.228.0> 2011-04-08 15:59:11 =============================================================================== moxi<0.228.0>: 2011-04-08 15:59:10: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13) moxi<0.228.0>: 2011-04-08 15:59:10: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <0.147.0> 2011-04-08 16:00:00 =============================================================================== ns_1@10.1.5.178:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303600,876735}}, {active_buckets,[]}, {memory, [{total,12240112}, {processes,2660508}, {processes_used,2656060}, {system,9579604}, {atom,488321}, {atom_used,455947}, {binary,42376}, {code,3855481}, {ets,327156}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,60}, {memory_data,{2075373568,1705984000,{<0.124.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 360484 kB\nBuffers: 246308 kB\nCached: 1247068 kB\nSwapCached: 0 kB\nActive: 680592 kB\nInactive: 887356 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 358848 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 40404 kB\nWriteback: 0 kB\nAnonPages: 74612 kB\nMapped: 26364 kB\nSlab: 85128 kB\nPageTables: 2132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 350904 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1276997632}, {buffered_memory,252219392}, {free_memory,369135616}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{50727,1}}, {context_switches,{13134,0}}, {garbage_collection,{3373,4911849,0}}, {io,{{input,4096162},{output,331997}}}, {reductions,{2129643,93874}}, {run_queue,0}, {runtime,{470,0}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:01:00 =============================================================================== ns_1@10.1.5.178:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303660,876751}}, {active_buckets,[]}, {memory, [{total,12298144}, {processes,2692812}, {processes_used,2688364}, {system,9605332}, {atom,488321}, {atom_used,455947}, {binary,48504}, {code,3855481}, {ets,345924}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,120}, {memory_data,{2075373568,1706110976,{<0.124.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 360608 kB\nBuffers: 246308 kB\nCached: 1247148 kB\nSwapCached: 0 kB\nActive: 680728 kB\nInactive: 887404 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 358972 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 30188 kB\nWriteback: 0 kB\nAnonPages: 74672 kB\nMapped: 26364 kB\nSlab: 84916 kB\nPageTables: 2132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 350908 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1277079552}, {buffered_memory,252219392}, {free_memory,369262592}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{110727,1}}, {context_switches,{17325,0}}, {garbage_collection,{4728,6484898,0}}, {io,{{input,4107773},{output,414799}}}, {reductions,{2761441,124382}}, {run_queue,0}, {runtime,{520,0}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:02:00 =============================================================================== ns_1@10.1.5.178:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303720,876752}}, {active_buckets,[]}, {memory, [{total,12327736}, {processes,2696804}, {processes_used,2691396}, {system,9630932}, {atom,488321}, {atom_used,455947}, {binary,54072}, {code,3855481}, {ets,364548}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,180}, {memory_data,{2075373568,1706237952,{<0.124.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 360484 kB\nBuffers: 246308 kB\nCached: 1247232 kB\nSwapCached: 0 kB\nActive: 680776 kB\nInactive: 887436 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 358848 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 26176 kB\nWriteback: 0 kB\nAnonPages: 74692 kB\nMapped: 26364 kB\nSlab: 84868 kB\nPageTables: 2132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 350908 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1277165568}, {buffered_memory,252219392}, {free_memory,369135616}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{170727,1}}, {context_switches,{21532,0}}, {garbage_collection,{6212,8051555,0}}, {io,{{input,4119384},{output,499619}}}, {reductions,{3393140,118540}}, {run_queue,0}, {runtime,{570,10}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:03:00 =============================================================================== ns_1@10.1.5.178:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303780,876757}}, {active_buckets,[]}, {memory, [{total,12269640}, {processes,2631660}, {processes_used,2626252}, {system,9637980}, {atom,488321}, {atom_used,455947}, {binary,43656}, {code,3855481}, {ets,381148}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,240}, {memory_data,{2075373568,1706491904,{<0.124.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 360360 kB\nBuffers: 246308 kB\nCached: 1247288 kB\nSwapCached: 0 kB\nActive: 680876 kB\nInactive: 887552 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 358724 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 17792 kB\nWriteback: 0 kB\nAnonPages: 74852 kB\nMapped: 26364 kB\nSlab: 84876 kB\nPageTables: 2132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 350908 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1277222912}, {buffered_memory,252219392}, {free_memory,369008640}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{230727,1}}, {context_switches,{26101,0}}, {garbage_collection,{7611,9691414,0}}, {io,{{input,4375450},{output,795576}}}, {reductions,{4046243,114401}}, {run_queue,0}, {runtime,{640,10}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:04:00 =============================================================================== ns_1@10.1.5.178:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303840,876742}}, {active_buckets,[]}, {memory, [{total,12310512}, {processes,2651516}, {processes_used,2646108}, {system,9658996}, {atom,488321}, {atom_used,455947}, {binary,47296}, {code,3855481}, {ets,397652}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,300}, {memory_data,{2075373568,1706618880,{<0.124.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 360236 kB\nBuffers: 246308 kB\nCached: 1247368 kB\nSwapCached: 0 kB\nActive: 680932 kB\nInactive: 887608 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 358600 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 9472 kB\nWriteback: 0 kB\nAnonPages: 74868 kB\nMapped: 26364 kB\nSlab: 84912 kB\nPageTables: 2132 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 350908 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1277304832}, {buffered_memory,252219392}, {free_memory,368881664}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{290727,1}}, {context_switches,{30351,0}}, {garbage_collection,{8961,11315152,0}}, {io,{{input,4387061},{output,881612}}}, {reductions,{4685447,115539}}, {run_queue,0}, {runtime,{700,10}}]}]}] INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== ********* RB: UNPRINTABLE REPORT ******** INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== ns_1@10.1.5.178:<0.64.0>:ns_cluster:98: engage_cluster(..) -> {ok,ok} INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== ns_1@10.1.5.178:<0.64.0>:ns_cluster:102: handling complete_join([{<<"targetNode">>, <<"ns_1@10.1.5.178">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 13140720}, {<<"usagePercent">>, 42}]}, {struct, [{<<"path">>, <<"/boot">>}, {<<"sizeKBytes">>, 101086}, {<<"usagePercent">>, 13}]}, {struct, [{<<"path">>, <<"/dev/shm">>}, {<<"sizeKBytes">>, 1005336}, {<<"usagePercent">>, 0}]}]}]}}, {<<"memoryQuota">>, 1570}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 26992074}, {<<"total">>, 2058928128}, {<<"quotaTotal">>, 1646264320}, {<<"used">>, 1567481856}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 2901256}, {<<"total">>, 13456097280.0}, {<<"quotaTotal">>, 13456097280.0}, {<<"used">>, 5651560857.0}, {<<"free">>, 7804536423.0}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>, []}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/var/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"uptime">>, <<"388">>}, {<<"memoryTotal">>, 2058928128}, {<<"memoryFree">>, 491446272}, {<<"mcdMemoryReserved">>, 1570}, {<<"mcdMemoryAllocated">>, 1570}, {<<"otpNode">>, <<"ns_1@10.1.5.192">>}, {<<"otpCookie">>, <<"lulxzmudqfmlvwvs">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.1.5.192:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.6.5.3_291_gba5fe27">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>, 11211}, {<<"direct">>, 11210}]}}]) INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== ns_log: logging ns_cluster:2:Node 'ns_1@10.1.5.178' is joining cluster via node 'ns_1@10.1.5.192'. INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: i_am_a_dead_man -> true ERROR REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ** gen_event handler {ns_pubsub,#Ref<0.0.0.681>} crashed. ** Was installed in ns_config_events ** Last event was: [{i_am_a_dead_man,true}] ** When handler state == {state,#Fun,ok} ** Reason == {noproc,{gen_server,call,[ns_moxi_sup,which_children,infinity]}} ERROR REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ** gen_event handler ns_port_init crashed. ** Was installed in ns_config_events ** Last event was: [{i_am_a_dead_man,true}] ** When handler state == {state} ** Reason == {noproc,{gen_server,call,[ns_port_sup,which_children,infinity]}} INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.136.0> 2011-04-08 16:04:10 =============================================================================== Pushing config INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== ns_cluster: joining cluster. Child has exited. INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: otp -> [{cookie,lulxzmudqfmlvwvs}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.178','ns_1@10.1.5.192'] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== pre-join cleaned config is: {config, {full,"/etc/membase/priv/config",undefined,ns_config_default}, [[{directory,"/var/membase/config"}], [{directory,"/opt/membase/1.7.0r/config"}, {nodes_wanted,['ns_1@10.1.5.178']}, {{node,'ns_1@10.1.5.178',membership},active}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@10.1.5.178',isasl},[{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {memory_quota,1583}, {buckets,[{configs,[]}]}, {{node,'ns_1@10.1.5.178',moxi},[{port,11211},{verbosity,[]}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,undefined}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up,server_joined, server_left,bucket_created,bucket_deleted, bucket_auth_failed]}]}, {replication,[{enabled,true}]}]], [[{nodes_wanted,['ns_1@10.1.5.178','ns_1@10.1.5.192']}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.178',moxi},[{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.178',isasl},[{path,"/var/membase/data/isasl.pw"}]}, {otp, [{'_vclock',[{'ns_1@10.1.5.178',{1,63469523050}}]}, {cookie,lulxzmudqfmlvwvs}]}]], ns_config_default, {ns_config,save_config_sync,[]}, <0.408.0>,true} INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== ns_1@10.1.5.178:<0.64.0>:ns_cluster:572: Connection from 'ns_1@10.1.5.178' to 'ns_1@10.1.5.192': true INFO REPORT <0.64.0> 2011-04-08 16:04:10 =============================================================================== ns_1@10.1.5.178:<0.64.0>:ns_cluster:581: Join status: {ok,ok}, starting ns_server_cluster back PROGRESS REPORT <0.414.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.415.0>}, {name,ns_log}, {mfa,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.416.0>}, {name,ns_log_events}, {mfa,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.417.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.418.0>}, {name,ns_node_disco_events}, {mfa,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.419.0> 2011-04-08 16:04:10 =============================================================================== Initting ns_node_disco with ['ns_1@10.1.5.192'] INFO REPORT <0.420.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.420.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.420.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192'], with cookie: lulxzmudqfmlvwvs PROGRESS REPORT <0.417.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.419.0>}, {name,ns_node_disco}, {mfa,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.417.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.428.0>}, {name,ns_node_disco_log}, {mfa,{ns_node_disco_log,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.417.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.429.0>}, {name,ns_node_disco_conf_events}, {mfa,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.430.0> 2011-04-08 16:04:10 =============================================================================== ns_config_rep init pulling INFO REPORT <0.430.0> 2011-04-08 16:04:10 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: alerts -> [{email,[]}, {email_alerts,false}, [{user,undefined}, {pass,"********"}, {addr,undefined}, {port,undefined}, {encrypt,false}], {alerts,[server_down,server_unresponsive,server_up,server_joined,server_left, bucket_created,bucket_deleted,bucket_auth_failed]}] INFO REPORT <0.430.0> 2011-04-08 16:04:10 =============================================================================== ns_config_rep init pushing INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== Writing isasl passwd file: "/var/membase/data/isasl.pw" INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: memory_quota -> 1570 INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193'] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: otp -> [{cookie,lulxzmudqfmlvwvs}] INFO REPORT <0.436.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: port_servers -> [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary","-r", "-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <0.437.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.438.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.436.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.430.0> 2011-04-08 16:04:10 =============================================================================== ns_config_rep init reannouncing INFO REPORT <0.439.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.437.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.438.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.439.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs PROGRESS REPORT <0.417.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.430.0>}, {name,ns_config_rep}, {mfa,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:10 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.417.0>}, {name,ns_node_disco_sup}, {mfa,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> inactiveAdded INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.192',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.192',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.192',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.192',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.192',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.192',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.193',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> inactiveAdded INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.193',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.193',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.193',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== config change: {node,'ns_1@10.1.5.193',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:10 =============================================================================== Pushing config done INFO REPORT <0.436.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.437.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.118.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:10 =============================================================================== Pushing config INFO REPORT <0.438.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.439.0> 2011-04-08 16:04:10 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.430.0> 2011-04-08 16:04:11 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: alerts -> [{email,[]}, {email_alerts,false}, [{user,undefined}, {pass,"********"}, {addr,undefined}, {port,undefined}, {encrypt,false}], {alerts,[server_down,server_unresponsive,server_up,server_joined,server_left, bucket_created,bucket_deleted,bucket_auth_failed]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: memory_quota -> 1570 INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193'] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: otp -> [{cookie,lulxzmudqfmlvwvs}] INFO REPORT <0.447.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: port_servers -> [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary","-r", "-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <0.448.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.449.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.447.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.450.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.448.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.449.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.450.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.178',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> inactiveAdded INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.178',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.178',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.178',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.178',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.192',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.192',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.192',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.192',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.192',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.192',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.193',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> inactiveAdded INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.193',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.193',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.193',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== config change: {node,'ns_1@10.1.5.193',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:11 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:11 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:11 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:11 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:12 =============================================================================== Pushing config done INFO REPORT <0.440.0> 2011-04-08 16:04:12 =============================================================================== ns_1@10.1.5.178:<0.440.0>:mb_master:93: Starting as candidate. Peers: ['ns_1@10.1.5.178', 'ns_1@10.1.5.192', 'ns_1@10.1.5.193'] INFO REPORT <0.440.0> 2011-04-08 16:04:12 =============================================================================== ns_1@10.1.5.178:<0.440.0>:mb_master:196: Changing master from undefined to 'ns_1@10.1.5.192' PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.440.0>}, {name,mb_master}, {mfa,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.457.0>}, {name,ns_heart}, {mfa,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.458.0>}, {name,ns_doctor}, {mfa,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.460.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.461.0>}, {name,menelaus_web}, {mfa,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.460.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.463.0>}, {name,menelaus_event}, {mfa,{menelaus_event,start_link,[]}}, {restart_type,transient}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.460.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.464.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.460.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.465.0>}, {name,menelaus_web_alerts_srv}, {mfa,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== ns_log: logging menelaus_app:1:Membase Server has started on web port 8091 on node 'ns_1@10.1.5.178'. PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.460.0>}, {name,menelaus}, {mfa,{menelaus_app,start_subapp,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.466.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.467.0>}, {name,ns_port_init}, {mfa,{ns_port_init,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.468.0> 2011-04-08 16:04:12 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.466.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.468.0>}, {name, {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.470.0> 2011-04-08 16:04:12 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.466.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.470.0>}, {name, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p","11210", "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.466.0>}, {name,ns_port_sup}, {mfa,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.472.0>}, {name,ns_tick_event}, {mfa,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.473.0>}, {name,ns_stats_event}, {mfa,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.474.0>}, {name,ns_bucket_worker}, {mfa,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.475.0>}, {name,ns_bucket_sup}, {mfa,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.477.0>}, {name,system_stats_collector}, {mfa,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.478.0>}, {name,{stats_archiver,"@system"}}, {mfa,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.480.0>}, {name,{stats_reader,"@system"}}, {mfa,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.481.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_moxi_sup} started [{pid,<0.482.0>}, {name,{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",[], 11220,8091}}, {mfa,{ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-B","auto","-z", "url=http://127.0.0.1:8091/pools/default/bucketsStreaming/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "-Z", "port_listen=11220,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-p","0","-Y","y","-O","stderr"], [use_stdio,stderr_to_stdout, {env,[{"MOXI_SASL_PLAIN_USR", "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {"MOXI_SASL_PLAIN_PWD",[]}]}]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.481.0>}, {name,ns_moxi_sup}, {mfa,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.483.0> 2011-04-08 16:04:12 =============================================================================== starting moxi_stats_collector with delay of 5000 PROGRESS REPORT <0.414.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.483.0>}, {name,moxi_stats_collector}, {mfa,{supervisor_cushion,start_link, [moxi_stats_collector,5000, moxi_stats_collector,start_link,[]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 16:04:12 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.414.0>}, {name,ns_server_sup}, {mfa,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.64.0> 2011-04-08 16:04:12 =============================================================================== ns_log: logging ns_cluster:3:Node ns_1@10.1.5.178 joined cluster INFO REPORT <0.64.0> 2011-04-08 16:04:12 =============================================================================== ns_1@10.1.5.178:<0.64.0>:ns_cluster:104: complete_join([{<<"targetNode">>, <<"ns_1@10.1.5.178">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 13140720}, {<<"usagePercent">>, 42}]}, {struct, [{<<"path">>, <<"/boot">>}, {<<"sizeKBytes">>, 101086}, {<<"usagePercent">>, 13}]}, {struct, [{<<"path">>, <<"/dev/shm">>}, {<<"sizeKBytes">>, 1005336}, {<<"usagePercent">>, 0}]}]}]}}, {<<"memoryQuota">>, 1570}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 26992074}, {<<"total">>, 2058928128}, {<<"quotaTotal">>, 1646264320}, {<<"used">>, 1567481856}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 2901256}, {<<"total">>, 13456097280.0}, {<<"quotaTotal">>, 13456097280.0}, {<<"used">>, 5651560857.0}, {<<"free">>, 7804536423.0}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>,[]}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/var/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"uptime">>, <<"388">>}, {<<"memoryTotal">>, 2058928128}, {<<"memoryFree">>, 491446272}, {<<"mcdMemoryReserved">>, 1570}, {<<"mcdMemoryAllocated">>, 1570}, {<<"otpNode">>, <<"ns_1@10.1.5.192">>}, {<<"otpCookie">>, <<"lulxzmudqfmlvwvs">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.1.5.192:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.6.5.3_291_gba5fe27">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>,11211}, {<<"direct">>, 11210}]}}]) -> {ok, ok} INFO REPORT <0.447.0> 2011-04-08 16:04:12 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.448.0> 2011-04-08 16:04:12 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.449.0> 2011-04-08 16:04:12 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.450.0> 2011-04-08 16:04:12 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.458.0> 2011-04-08 16:04:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:82: Got initial status [{'ns_1@10.1.5.178', [{last_heard, {1302,303852, 329742}}, {active_buckets, []}, {memory, [{total, 13748912}, {processes, 3931072}, {processes_used, 3927888}, {system, 9817840}, {atom,489289}, {atom_used, 457642}, {binary, 143480}, {code,3877338}, {ets,402260}]}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.13.4"}, {mnesia, "4.4.12"}, {stdlib, "1.16.4"}, {os_mon, "2.2.4"}, {ns_server, "1.6.5.3_291_gba5fe27"}, {menelaus, "1.6.5.3_291_gba5fe27"}, {sasl, "2.1.8"}]}, {system_arch, "i686-pc-linux-gnu"}, {wall_clock,312}, {memory_data, {2075373568, 1706618880, {<0.124.0>, 786412}}}, {disk_data, [{"/",11173104, 41}, {"/boot", 101086,13}, {"/dev/shm", 1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 359864 kB\nBuffers: 246308 kB\nCached: 1247488 kB\nSwapCached: 0 kB\nActive: 681864 kB\nInactive: 887668 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 358228 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 9688 kB\nWriteback: 16 kB\nAnonPages: 75724 kB\nMapped: 26156 kB\nSlab: 84864 kB\nPageTables: 2080 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 314416 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", 1.0}]}, {system_memory_data, [{system_total_memory, 2075373568}, {free_swap, 4160741376}, {total_swap, 4160741376}, {cached_memory, 1277427712}, {buffered_memory, 252219392}, {free_memory, 369262592}, {total_memory, 2075373568}]}, {statistics, [{wall_clock, {311990, 1812}}, {context_switches, {33156,0}}, {garbage_collection, {9623, 12770448,0}}, {io, {{input, 4479714}, {output, 1010577}}}, {reductions, {5271423, 377012}}, {run_queue,1}, {runtime, {810,80}}]}]}, {'ns_1@10.1.5.192', [{last_heard, {1302,303852, 329774}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total, 31952784}, {processes, 11747816}, {processes_used, 11730016}, {system, 20204968}, {atom,820249}, {atom_used, 812647}, {binary, 254760}, {code,8534778}, {ets, 1238840}]}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.13.4"}, {mnesia, "4.4.12"}, {stdlib, "1.16.4"}, {os_mon, "2.2.4"}, {inets,"5.2"}, {ns_server, "1.6.5.3_291_gba5fe27"}, {menelaus, "1.6.5.3_291_gba5fe27"}, {sasl, "2.1.8"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,390}, {memory_data, {2058928128, 1567481856, {<10463.124.0>, 1572784}}}, {disk_data, [{"/",13140720, 42}, {"/boot", 101086,13}, {"/dev/shm", 1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 476332 kB\nBuffers: 352548 kB\nCached: 894068 kB\nSwapCached: 17552 kB\nActive: 756040 kB\nInactive: 629276 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 476332 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 11440 kB\nWriteback: 0 kB\nAnonPages: 134448 kB\nMapped: 17488 kB\nSlab: 124376 kB\nPageTables: 6424 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 563636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", 1.0}]}, {system_memory_data, [{system_total_memory, 2058928128}, {free_swap, 2014429184}, {total_swap, 2080366592}, {cached_memory, 915525632}, {buffered_memory, 361009152}, {free_memory, 487763968}, {total_memory, 2058928128}]}, {statistics, [{wall_clock, {381306,84}}, {context_switches, {57990,0}}, {garbage_collection, {11907, 30931572,0}}, {io, {{input, 5663618}, {output, 2599439}}}, {reductions, {15620061, 1896163}}, {run_queue,0}, {runtime, {2240, 170}}]}]}, {'ns_1@10.1.5.193', [{last_heard, {1302,303852, 329776}}, {active_buckets, []}, {memory, [{total, 28540376}, {processes, 10039048}, {processes_used, 10033760}, {system, 18501328}, {atom,730529}, {atom_used, 698792}, {binary,87520}, {code,7582094}, {ets,801256}]}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.13.4"}, {mnesia, "4.4.12"}, {stdlib, "1.16.4"}, {os_mon, "2.2.4"}, {ns_server, "1.6.5.3_291_gba5fe27"}, {menelaus, "1.6.5.3_291_gba5fe27"}, {sasl, "2.1.8"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,357}, {memory_data, {2058928128, 993751040, {<10507.124.0>, 1572784}}}, {disk_data, [{"/",13140720, 39}, {"/boot", 101086,13}, {"/dev/shm", 1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1035372 kB\nBuffers: 252624 kB\nCached: 505220 kB\nSwapCached: 17896 kB\nActive: 610812 kB\nInactive: 229776 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1035372 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 10564 kB\nWriteback: 0 kB\nAnonPages: 78192 kB\nMapped: 20004 kB\nSlab: 111216 kB\nPageTables: 5964 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 451316 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", 1.0}]}, {system_memory_data, [{system_total_memory, 2058928128}, {free_swap, 2031652864}, {total_swap, 2080366592}, {cached_memory, 517345280}, {buffered_memory, 258686976}, {free_memory, 1060220928}, {total_memory, 2058928128}]}, {statistics, [{wall_clock, {354806,2}}, {context_switches, {36614,0}}, {garbage_collection, {7295, 11784378,0}}, {io, {{input, 4517237}, {output, 1088227}}}, {reductions, {5213181, 155170}}, {run_queue,0}, {runtime, {1080, 30}}]}]}] INFO REPORT <0.469.0> 2011-04-08 16:04:13 =============================================================================== moxi<0.469.0>: 2011-04-08 16:04:12: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13) moxi<0.469.0>: 2011-04-08 16:04:12: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <0.482.0> 2011-04-08 16:04:14 =============================================================================== moxi<0.482.0>: 2011-04-08 16:04:12: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (55) INFO REPORT <0.471.0> 2011-04-08 16:04:14 =============================================================================== ns_log: logging ns_port_server:0:Port server memcached on node 'ns_1@10.1.5.178' exited with status 137. Restarting. Messages: ERROR REPORT <0.471.0> 2011-04-08 16:04:14 =============================================================================== ** Generic server <0.471.0> terminating ** Last message in was {#Port<0.2390>,{exit_status,137}} ** When Server state == {state,#Port<0.2390>,memcached, {[empty],[empty,empty]}, undefined,[],0} ** Reason for termination == ** {abnormal,137} CRASH REPORT <0.471.0> 2011-04-08 16:04:14 =============================================================================== Crashing process initial_call {ns_port_server,init,['Argument__1']} pid <0.471.0> registered_name [] error_info {exit,{abnormal,137}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [<0.470.0>,ns_port_sup,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [{'EXIT',#Port<0.2390>,normal}] links [<0.470.0>] dictionary [] trap_exit true status running heap_size 610 stack_size 24 reductions 690 INFO REPORT <0.470.0> 2011-04-08 16:04:14 =============================================================================== Cushion managed supervisor for memcached failed: {abnormal,137} INFO REPORT <0.470.0> 2011-04-08 16:04:14 =============================================================================== ns_log: logging supervisor_cushion:1:Service memcached exited on node 'ns_1@10.1.5.178' in 2.53s ERROR REPORT <0.470.0> 2011-04-08 16:04:19 =============================================================================== ** Generic server <0.470.0> terminating ** Last message in was {die,{error,cushioned_supervisor,{abnormal,137}}} ** When Server state == {state,memcached,5000,{1302,303852,138435},undefined} ** Reason for termination == ** {error,cushioned_supervisor,{abnormal,137}} CRASH REPORT <0.470.0> 2011-04-08 16:04:19 =============================================================================== Crashing process initial_call {supervisor_cushion,init,['Argument__1']} pid <0.470.0> registered_name [] error_info {exit,{error,cushioned_supervisor,{abnormal,137}}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_port_sup,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [] links [<0.466.0>] dictionary [] trap_exit true status running heap_size 610 stack_size 24 reductions 561 SUPERVISOR REPORT <0.466.0> 2011-04-08 16:04:19 =============================================================================== Reporting supervisor {local,ns_port_sup} Child process errorContext child_terminated reason {error,cushioned_supervisor,{abnormal,137}} pid <0.470.0> name {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]} start_function {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so", "-p","11210","-E", "./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]} restart_type permanent shutdown 10 child_type worker INFO REPORT <0.488.0> 2011-04-08 16:04:19 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.466.0> 2011-04-08 16:04:19 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.488.0>}, {name, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p","11210", "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== config change: rebalance_status -> running INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config done INFO REPORT <0.474.0> 2011-04-08 16:04:24 =============================================================================== ns_1@10.1.5.178:<0.474.0>:ns_bucket_sup:79: Starting new child: {{per_bucket_sup, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {single_bucket_sup, start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, permanent, infinity, supervisor, [single_bucket_sup]} INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config PROGRESS REPORT <0.475.0> 2011-04-08 16:04:24 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<0.510.0>}, {name,{per_bucket_sup,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{single_bucket_sup,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:24 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:24 =============================================================================== Pushing config done INFO REPORT <0.512.0> 2011-04-08 16:04:25 =============================================================================== ns_1@10.1.5.178:<0.512.0>:ns_memcached:438: Created bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" with config string "vb0=false;waitforwarmup=false;ht_size=3079;ht_locks=5;failpartialwarmup=false;db_shards=4;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB;tap_noop_interval=20;max_txn_size=1000;max_size=419430400;initfile=/etc/membase/priv/init.sql;tap_keepalive=300;dbname=/var/membase/data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561;" INFO REPORT <0.512.0> 2011-04-08 16:04:26 =============================================================================== ns_log: logging ns_memcached:1:Bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" loaded on node 'ns_1@10.1.5.178' in 1 seconds. PROGRESS REPORT <0.511.0> 2011-04-08 16:04:26 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.512.0>}, {name,{ns_memcached,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{ns_memcached,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <0.511.0> 2011-04-08 16:04:26 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.518.0>}, {name,{ns_vbm_sup,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{ns_vbm_sup,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <0.511.0> 2011-04-08 16:04:26 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.519.0>}, {name,{stats_collector,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{stats_collector,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.511.0> 2011-04-08 16:04:26 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.520.0>}, {name,{stats_archiver,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{stats_archiver,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute' PROGRESS REPORT <0.511.0> 2011-04-08 16:04:26 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.521.0>}, {name,{stats_reader,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{stats_reader,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,692139},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,818,<0.523.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,692139},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,818,<0.523.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour' INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,707129},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,819,<0.530.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,707129},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,819,<0.530.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day' INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,723080},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,820,<0.537.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,723080},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,820,<0.537.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week' INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,738098},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,821,<0.544.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,738098},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,821,<0.544.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month' INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,743973},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,822,<0.551.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,743973},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,822,<0.551.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year' INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,749189},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,823,<0.558.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:26 =============================================================================== ns_1@10.1.5.178:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.178']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303866,749189},'ns_1@10.1.5.178'}}, {version,{{2,0},[]}}]}, {tid,823,<0.558.0>}} INFO REPORT <0.519.0> 2011-04-08 16:04:27 =============================================================================== ns_1@10.1.5.178:<0.519.0>:stats_collector:80: Stats for bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561": auth_cmds 0 auth_errors 0 bucket_conns 1 bytes_read 103 bytes_written 15680 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 11 curr_connections 11 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /var/membase/data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561 ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 0 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 419430400 ep_max_txn_size 1000 ep_mem_high_wat 314572800 ep_mem_low_wat 251658239 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs0 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 344 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.6.5.3_143_g366c22e ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 14274 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 1.4.13-stable limit_maxbytes 67108864 mem_used 344 pid 10641 pointer_size 32 proxy_cmd_count 0 proxy_cmd_time 0 proxy_local_cmd_count 0 proxy_local_cmd_time 0 rejected_conns 0 rusage_system 0.034994 rusage_user 0.569913 threads 4 time 1302303866 total_connections 11 uptime 9 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_430_g6235145 INFO REPORT <0.118.0> 2011-04-08 16:04:27 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}, {fastForwardMap,[['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178'|...], [...]|...]}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:04:27 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:27 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:27 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:27 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:04:27 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:04:27 =============================================================================== Pushing config done INFO REPORT <0.430.0> 2011-04-08 16:04:39 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.118.0> 2011-04-08 16:05:10 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:05:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:05:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:05:10 =============================================================================== config change: rebalance_status -> none INFO REPORT <0.118.0> 2011-04-08 16:05:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:05:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:05:10 =============================================================================== Pushing config done INFO REPORT <0.458.0> 2011-04-08 16:05:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303911,63489}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,20073312}, {processes,9766788}, {processes_used,9756668}, {system,10306524}, {atom,519117}, {atom_used,500042}, {binary,230872}, {code,4055640}, {ets,590652}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,370}, {memory_data,{2075373568,1756721152,{<0.512.0>,3814172}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 311060 kB\nBuffers: 246336 kB\nCached: 1250260 kB\nSwapCached: 0 kB\nActive: 731192 kB\nInactive: 886392 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 309424 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 1344 kB\nWriteback: 0 kB\nAnonPages: 120996 kB\nMapped: 29168 kB\nSlab: 85356 kB\nPageTables: 2276 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 520096 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1280266240}, {buffered_memory,252248064}, {free_memory,318525440}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{361916,1}}, {context_switches,{49169,0}}, {garbage_collection,{14840,82639957,0}}, {io,{{input,5850936},{output,2259304}}}, {reductions,{20141168,5745449}}, {run_queue,0}, {runtime,{4960,1630}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,303911,145622}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34746344}, {processes,14225896}, {processes_used,14208584}, {system,20520448}, {atom,824289}, {atom_used,817419}, {binary,202504}, {code,8617057}, {ets,1438960}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,449}, {memory_data,{2058928128,1576886272,{<10463.124.0>,1572784}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 468884 kB\nBuffers: 352548 kB\nCached: 895180 kB\nSwapCached: 17552 kB\nActive: 762696 kB\nInactive: 629816 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 468884 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 3272 kB\nWriteback: 0 kB\nAnonPages: 140476 kB\nMapped: 17604 kB\nSlab: 124664 kB\nPageTables: 6512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 583960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,916664320}, {buffered_memory,361009152}, {free_memory,480137216}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{441245,17}}, {context_switches,{98085,0}}, {garbage_collection,{25964,96347126,0}}, {io,{{input,6413196},{output,4587339}}}, {reductions,{30308815,2258660}}, {run_queue,0}, {runtime,{4590,490}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,303911,323948}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,33888824}, {processes,14464376}, {processes_used,14447912}, {system,19424448}, {atom,784585}, {atom_used,765250}, {binary,239256}, {code,7930444}, {ets,1164328}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,417}, {memory_data,{2058928128,1000730624,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 978244 kB\nBuffers: 252624 kB\nCached: 508064 kB\nSwapCached: 17896 kB\nActive: 669020 kB\nInactive: 228484 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 978244 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 2196 kB\nWriteback: 0 kB\nAnonPages: 132216 kB\nMapped: 20228 kB\nSlab: 111404 kB\nPageTables: 5928 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 544928 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,520257536}, {buffered_memory,258686976}, {free_memory,1001721856}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{414838,31}}, {context_switches,{50266,0}}, {garbage_collection,{11004,46649900,0}}, {io,{{input,5577494},{output,2172295}}}, {reductions,{13155387,4083762}}, {run_queue,0}, {runtime,{3340,1280}}]}]}] PROGRESS REPORT <0.518.0> 2011-04-08 16:05:13 =============================================================================== supervisor {local,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.640.0>}, {name, {child_id, [170,169,168,167,166,165,164,163,162,161,160,159,158,157, 156,155,154,153,152,151,150,149,148,147,146,145,144,143, 142,141,140,139,138,137,136,135,134,133,132,131,130,129, 128,127,126,125,124,123,122,121,120,119,118,117,116,115, 114,113,112,111,110,109,108,107,106,105,104,103,102,101, 100,99,98,97,96,95,94,93,92,91,90,89,88,87,86,85,84,83,82, 81,80,79,78,77,76,75,74,73,72,71,70,69,68,67,66,65,64,63, 62,61,60,59,58,57,56,55,54,53,52,51,50,49,48,47,46,45,44, 43,42,41,40,39,38,37,36,35,34,33,32,31,30,29,28,27,26,25, 24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3, 2,1,0], 'ns_1@10.1.5.192'}}, {mfa, {ebucketmigrator_srv,start_link, [{"10.1.5.178",11210}, {"10.1.5.192",11210}, [{username, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {password,[]}, {vbuckets, [170,169,168,167,166,165,164,163,162,161,160,159,158, 157,156,155,154,153,152,151,150,149,148,147,146,145, 144,143,142,141,140,139,138,137,136,135,134,133,132, 131,130,129,128,127,126,125,124,123,122,121,120,119, 118,117,116,115,114,113,112,111,110,109,108,107,106, 105,104,103,102,101,100,99,98,97,96,95,94,93,92,91, 90,89,88,87,86,85,84,83,82,81,80,79,78,77,76,75,74, 73,72,71,70,69,68,67,66,65,64,63,62,61,60,59,58,57, 56,55,54,53,52,51,50,49,48,47,46,45,44,43,42,41,40, 39,38,37,36,35,34,33,32,31,30,29,28,27,26,25,24,23, 22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3, 2,1,0]}, {takeover,false}, {suffix, "ns_1@10.1.5.192-ns_1@10.1.5.192-1302303903.309329"}]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.518.0> 2011-04-08 16:05:13 =============================================================================== supervisor {local,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.641.0>}, {name, {child_id, [341,340,339,338,337,336,335,334,333,332,331,330,329,328, 327,326,325,324,323,322,321,320,319,318,317,316,315,314, 313,312,311,310,309,308,307,306,305,304,303,302,301,300, 299,298,297,296,295,294,293,292,291,290,289,288,287,286, 285,284,283,282,281,280,279,278,277,276,275,274,273,272, 271,270,269,268,267,266,265,264,263,262,261,260,259,258, 257,256,255,254,253,252,251,250,249,248,247,246,245,244, 243,242,241,240,239,238,237,236,235,234,233,232,231,230, 229,228,227,226,225,224,223,222,221,220,219,218,217,216, 215,214,213,212,211,210,209,208,207,206,205,204,203,202, 201,200,199,198,197,196,195,194,193,192,191,190,189,188, 187,186,185,184,183,182,181,180,179,178,177,176,175,174, 173,172,171], 'ns_1@10.1.5.193'}}, {mfa, {ebucketmigrator_srv,start_link, [{"10.1.5.178",11210}, {"10.1.5.193",11210}, [{username, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {password,[]}, {vbuckets, [341,340,339,338,337,336,335,334,333,332,331,330,329, 328,327,326,325,324,323,322,321,320,319,318,317,316, 315,314,313,312,311,310,309,308,307,306,305,304,303, 302,301,300,299,298,297,296,295,294,293,292,291,290, 289,288,287,286,285,284,283,282,281,280,279,278,277, 276,275,274,273,272,271,270,269,268,267,266,265,264, 263,262,261,260,259,258,257,256,255,254,253,252,251, 250,249,248,247,246,245,244,243,242,241,240,239,238, 237,236,235,234,233,232,231,230,229,228,227,226,225, 224,223,222,221,220,219,218,217,216,215,214,213,212, 211,210,209,208,207,206,205,204,203,202,201,200,199, 198,197,196,195,194,193,192,191,190,189,188,187,186, 185,184,183,182,181,180,179,178,177,176,175,174,173, 172,171]}, {takeover,false}, {suffix, "ns_1@10.1.5.193-ns_1@10.1.5.192-1302303903.443845"}]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.430.0> 2011-04-08 16:05:24 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.430.0> 2011-04-08 16:05:46 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.519.0> 2011-04-08 16:06:07 =============================================================================== ns_1@10.1.5.178:<0.519.0>:stats_collector:80: Stats for bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561": auth_cmds 347 auth_errors 0 bucket_conns 6 bytes_read 1011498 bytes_written 40545035 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 34 conn_yields 16 connection_structures 16 curr_connections 16 curr_items 44 curr_items_tot 89 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 63 ep_commit_time 0 ep_commit_time_total 10 ep_data_age 0 ep_data_age_highwat 4 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /var/membase/data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561 ep_dbshards 4 ep_diskqueue_drain 89 ep_diskqueue_fill 89 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_duration 0 ep_flush_duration_highwat 3 ep_flush_duration_total 10 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 89 ep_io_read_bytes 0 ep_io_write_bytes 915173 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 921567 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 34 ep_max_data_size 419430400 ep_max_txn_size 1000 ep_mem_high_wat 314572800 ep_mem_low_wat 251658239 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs20 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 8953125 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 4 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 920883 ep_total_del_items 0 ep_total_enqueued 89 ep_total_new_items 89 ep_total_persisted 89 ep_vb_total 684 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.6.5.3_143_g366c22e ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 14274 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 1.4.13-stable limit_maxbytes 67108864 mem_used 9874692 pid 10641 pointer_size 32 proxy_cmd_count 0 proxy_cmd_time 0 proxy_local_cmd_count 0 proxy_local_cmd_time 0 rejected_conns 0 rusage_system 0.586910 rusage_user 4.425327 tap_connect_received 2 tap_mutation_received 55 tap_mutation_sent 37 tap_opaque_received 686 tap_opaque_sent 344 tap_vbucket_set_received 684 threads 4 time 1302303966 total_connections 358 uptime 109 vb_active_curr_items 44 vb_active_eject 0 vb_active_ht_memory 4287312 vb_active_itm_memory 455268 vb_active_num 342 vb_active_num_non_resident 0 vb_active_ops_create 44 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 44 vb_active_queue_fill 44 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 45 vb_replica_eject 0 vb_replica_ht_memory 4287312 vb_replica_itm_memory 465615 vb_replica_num 342 vb_replica_num_non_resident 0 vb_replica_ops_create 45 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 45 vb_replica_queue_fill 45 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_430_g6235145 INFO REPORT <0.458.0> 2011-04-08 16:06:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303971,64939}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,18883232}, {processes,8180292}, {processes_used,8171196}, {system,10702940}, {atom,519117}, {atom_used,500441}, {binary,489296}, {code,4062708}, {ets,718260}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,430}, {memory_data,{2075373568,1766633472,{<0.512.0>,1457156}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 301380 kB\nBuffers: 246336 kB\nCached: 1251948 kB\nSwapCached: 0 kB\nActive: 739128 kB\nInactive: 887580 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 299744 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 624 kB\nWriteback: 0 kB\nAnonPages: 128428 kB\nMapped: 29228 kB\nSlab: 85376 kB\nPageTables: 2284 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 526124 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1281994752}, {buffered_memory,252248064}, {free_memory,308613120}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{422266,351}}, {context_switches,{75755,0}}, {garbage_collection,{37298,261976822,0}}, {io,{{input,6288543},{output,3594080}}}, {reductions,{58188622,6377475}}, {run_queue,0}, {runtime,{15530,1770}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,303971,146731}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,37518752}, {processes,16796240}, {processes_used,16776792}, {system,20722512}, {atom,824289}, {atom_used,817742}, {binary,242680}, {code,8617057}, {ets,1599080}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,509}, {memory_data,{2058928128,1585283072,{<10463.1839.0>,3144104}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 462792 kB\nBuffers: 352548 kB\nCached: 899468 kB\nSwapCached: 17552 kB\nActive: 768144 kB\nInactive: 630376 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 462792 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 492 kB\nWriteback: 0 kB\nAnonPages: 142252 kB\nMapped: 17612 kB\nSlab: 124660 kB\nPageTables: 6528 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 576228 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,921055232}, {buffered_memory,361009152}, {free_memory,473899008}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{501245,23}}, {context_switches,{127145,0}}, {garbage_collection,{33715,138787164,0}}, {io,{{input,10080881},{output,6744949}}}, {reductions,{52110615,3850984}}, {run_queue,0}, {runtime,{6880,350}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,303971,325463}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35739104}, {processes,15961912}, {processes_used,15947320}, {system,19777192}, {atom,784585}, {atom_used,765809}, {binary,333776}, {code,7944128}, {ets,1403032}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,477}, {memory_data,{2058928128,1065332736,{<10507.569.0>,2543288}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 963844 kB\nBuffers: 252624 kB\nCached: 509772 kB\nSwapCached: 17896 kB\nActive: 682164 kB\nInactive: 228928 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 963844 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 244 kB\nWriteback: 0 kB\nAnonPages: 144136 kB\nMapped: 20300 kB\nSlab: 111532 kB\nPageTables: 5960 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 561224 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,522006528}, {buffered_memory,258686976}, {free_memory,986976256}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{474974,168}}, {context_switches,{76396,0}}, {garbage_collection,{33130,217708564,0}}, {io,{{input,6029391},{output,3590784}}}, {reductions,{49247895,6113912}}, {run_queue,0}, {runtime,{14430,1850}}]}]}] INFO REPORT <0.430.0> 2011-04-08 16:06:28 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.430.0> 2011-04-08 16:06:41 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.430.0> 2011-04-08 16:06:55 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.118.0> 2011-04-08 16:07:09 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:07:09 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:09 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:09 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:09 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:09 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:09 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> inactiveFailed INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config done INFO REPORT <0.474.0> 2011-04-08 16:07:10 =============================================================================== ns_1@10.1.5.178:<0.474.0>:ns_bucket_sup:86: Stopping child for dead bucket: {{per_bucket_sup, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, <0.510.0>, supervisor, [single_bucket_sup]} INFO REPORT <0.510.0> 2011-04-08 16:07:10 =============================================================================== ns_1@10.1.5.178:<0.510.0>:single_bucket_sup:27: Delegating exit {'EXIT', <0.475.0>, shutdown} to child supervisor: <0.511.0> INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config INFO REPORT <0.512.0> 2011-04-08 16:07:10 =============================================================================== ns_log: logging ns_memcached:2:Shutting down bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" on 'ns_1@10.1.5.178' for server shutdown INFO REPORT <0.510.0> 2011-04-08 16:07:10 =============================================================================== ns_1@10.1.5.178:<0.510.0>:single_bucket_sup:24: per-bucket supervisor for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" died with reason shutdown INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> inactiveFailed INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:10 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:10 =============================================================================== Pushing config done INFO REPORT <0.458.0> 2011-04-08 16:07:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304031,66460}}, {active_buckets,[]}, {memory, [{total,15165592}, {processes,4648332}, {processes_used,4635276}, {system,10517260}, {atom,519117}, {atom_used,500441}, {binary,222688}, {code,4062708}, {ets,800500}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,490}, {memory_data,{2075373568,1768157184,{<0.512.0>,1457156}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 300016 kB\nBuffers: 246336 kB\nCached: 1253020 kB\nSwapCached: 0 kB\nActive: 740228 kB\nInactive: 888380 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 298380 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 268 kB\nWriteback: 0 kB\nAnonPages: 129240 kB\nMapped: 29228 kB\nSlab: 85380 kB\nPageTables: 2288 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 525940 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1283092480}, {buffered_memory,252248064}, {free_memory,307216384}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{482263,349}}, {context_switches,{102669,0}}, {garbage_collection,{71212,454080930,0}}, {io,{{input,6590731},{output,4544911}}}, {reductions,{99358148,6986496}}, {run_queue,0}, {runtime,{26710,1910}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304031,140094}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,33317800}, {processes,12548312}, {processes_used,12525520}, {system,20769488}, {atom,824289}, {atom_used,817742}, {binary,133264}, {code,8617057}, {ets,1753952}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,569}, {memory_data,{2058928128,1584140288,{<10463.1839.0>,3145928}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 465520 kB\nBuffers: 352552 kB\nCached: 898652 kB\nSwapCached: 17552 kB\nActive: 765864 kB\nInactive: 629752 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 465520 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 368 kB\nWriteback: 0 kB\nAnonPages: 140152 kB\nMapped: 17612 kB\nSlab: 124592 kB\nPageTables: 6516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 574876 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,920219648}, {buffered_memory,361013248}, {free_memory,476692480}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{561243,21}}, {context_switches,{147374,0}}, {garbage_collection,{39850,173290061,0}}, {io,{{input,15998244},{output,8576868}}}, {reductions,{74425948,3210124}}, {run_queue,0}, {runtime,{8930,250}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304031,305459}}, {active_buckets,[]}, {memory, [{total,29462880}, {processes,9655912}, {processes_used,9633832}, {system,19806968}, {atom,784585}, {atom_used,765809}, {binary,216488}, {code,7944128}, {ets,1551784}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,537}, {memory_data,{2058928128,1072967680,{<10507.559.0>,2914272}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 963224 kB\nBuffers: 252624 kB\nCached: 510768 kB\nSwapCached: 17896 kB\nActive: 682572 kB\nInactive: 229464 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 963224 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 184 kB\nWriteback: 0 kB\nAnonPages: 144068 kB\nMapped: 20300 kB\nSlab: 111508 kB\nPageTables: 5956 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 561224 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523026432}, {buffered_memory,258686976}, {free_memory,986341376}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{534986,181}}, {context_switches,{103071,0}}, {garbage_collection,{59521,405432890,0}}, {io,{{input,6330601},{output,4430562}}}, {reductions,{89265358,6714663}}, {run_queue,0}, {runtime,{26370,1980}}]}]}] INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.118.0> 2011-04-08 16:07:25 =============================================================================== config change: rebalance_status -> running INFO REPORT <0.118.0> 2011-04-08 16:07:25 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:25 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config done INFO REPORT <0.474.0> 2011-04-08 16:07:25 =============================================================================== ns_1@10.1.5.178:<0.474.0>:ns_bucket_sup:79: Starting new child: {{per_bucket_sup, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {single_bucket_sup, start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, permanent, infinity, supervisor, [single_bucket_sup]} PROGRESS REPORT <0.475.0> 2011-04-08 16:07:25 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<0.765.0>}, {name,{per_bucket_sup,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{single_bucket_sup,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.118.0> 2011-04-08 16:07:25 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:07:25 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:25 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:25 =============================================================================== Pushing config done INFO REPORT <0.767.0> 2011-04-08 16:07:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:07:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.464.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:07:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.776.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:07:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:07:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.776.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:07:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.779.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:07:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.118.0> 2011-04-08 16:07:36 =============================================================================== config change: rebalance_status -> {none,<<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>} INFO REPORT <0.118.0> 2011-04-08 16:07:36 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:36 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:36 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:36 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.430.0> 2011-04-08 16:07:36 =============================================================================== Pushing config INFO REPORT <0.430.0> 2011-04-08 16:07:36 =============================================================================== Pushing config done INFO REPORT <0.767.0> 2011-04-08 16:07:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:07:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.779.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:07:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.788.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:07:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:07:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.788.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:07:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.791.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:07:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:07:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.791.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:07:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.793.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:07:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:07:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.793.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:07:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.796.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:07:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:07:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.796.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:07:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.798.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:07:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:07:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:08:01 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:08:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.798.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.822.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.822.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.824.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:08:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304091,65549}}, {active_buckets,[]}, {memory, [{total,15260072}, {processes,4736268}, {processes_used,4724388}, {system,10523804}, {atom,519117}, {atom_used,500490}, {binary,209520}, {code,4062708}, {ets,817684}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,550}, {memory_data,{2075373568,1766633472,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302868 kB\nBuffers: 246344 kB\nCached: 1251860 kB\nSwapCached: 0 kB\nActive: 737784 kB\nInactive: 887472 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 301232 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 192 kB\nWriteback: 0 kB\nAnonPages: 127048 kB\nMapped: 28212 kB\nSlab: 85372 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518932 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1281904640}, {buffered_memory,252256256}, {free_memory,310136832}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{541919,2}}, {context_switches,{112308,0}}, {garbage_collection,{77743,484711055,0}}, {io,{{input,8820296},{output,5802199}}}, {reductions,{105876373,218018}}, {run_queue,0}, {runtime,{28390,70}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304091,140518}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35394736}, {processes,14483952}, {processes_used,14462016}, {system,20910784}, {atom,828329}, {atom_used,823498}, {binary,123304}, {code,8669006}, {ets,1844696}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,629}, {memory_data,{2058928128,1582489600,{<10463.2477.0>,2543360}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 465512 kB\nBuffers: 352552 kB\nCached: 899692 kB\nSwapCached: 17552 kB\nActive: 765152 kB\nInactive: 630324 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 465512 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 516 kB\nWriteback: 0 kB\nAnonPages: 138988 kB\nMapped: 17612 kB\nSlab: 124588 kB\nPageTables: 6504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 570556 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,921284608}, {buffered_memory,361013248}, {free_memory,476684288}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{621236,14}}, {context_switches,{166562,0}}, {garbage_collection,{45341,202292221,0}}, {io,{{input,16480555},{output,10224228}}}, {reductions,{87776300,1195449}}, {run_queue,0}, {runtime,{10410,180}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304091,305912}}, {active_buckets,[]}, {memory, [{total,29824672}, {processes,9990232}, {processes_used,9968760}, {system,19834440}, {atom,784585}, {atom_used,765874}, {binary,206032}, {code,7944128}, {ets,1585032}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,597}, {memory_data,{2058928128,1068777472,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966704 kB\nBuffers: 252628 kB\nCached: 510884 kB\nSwapCached: 17896 kB\nActive: 678400 kB\nInactive: 229792 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966704 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 212 kB\nWriteback: 0 kB\nAnonPages: 140124 kB\nMapped: 20308 kB\nSlab: 111580 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523145216}, {buffered_memory,258691072}, {free_memory,989904896}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{594806,2}}, {context_switches,{110192,0}}, {garbage_collection,{62306,414948271,0}}, {io,{{input,8910935},{output,6998814}}}, {reductions,{91370629,152270}}, {run_queue,0}, {runtime,{26950,30}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.824.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.828.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.828.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.830.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.830.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.833.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:08:23 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:08:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.833.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.835.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.835.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.838.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.838.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.840.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.840.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.843.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.843.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.845.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.845.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.848.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:08:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.848.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:08:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.850.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:08:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:08:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.850.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.853.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.853.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.855.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:09:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304151,65028}}, {active_buckets,[]}, {memory, [{total,15277336}, {processes,4770180}, {processes_used,4758300}, {system,10507156}, {atom,519117}, {atom_used,500490}, {binary,208344}, {code,4062708}, {ets,801284}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,610}, {memory_data,{2075373568,1765498880,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302612 kB\nBuffers: 246344 kB\nCached: 1251968 kB\nSwapCached: 0 kB\nActive: 737852 kB\nInactive: 887532 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300976 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 1932 kB\nWriteback: 0 kB\nAnonPages: 127068 kB\nMapped: 28212 kB\nSlab: 85428 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518932 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282015232}, {buffered_memory,252256256}, {free_memory,309874688}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{601916,1}}, {context_switches,{117196,0}}, {garbage_collection,{78607,487151617,0}}, {io,{{input,9062955},{output,5956967}}}, {reductions,{106619586,125151}}, {run_queue,0}, {runtime,{28490,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304151,141339}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34187272}, {processes,13124840}, {processes_used,13102824}, {system,21062432}, {atom,828329}, {atom_used,823498}, {binary,120696}, {code,8669006}, {ets,1997328}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,689}, {memory_data,{2058928128,1584140288,{<10463.2477.0>,2543552}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 463288 kB\nBuffers: 352552 kB\nCached: 900456 kB\nSwapCached: 17552 kB\nActive: 766948 kB\nInactive: 630876 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 463288 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 136 kB\nWriteback: 0 kB\nAnonPages: 140592 kB\nMapped: 17612 kB\nSlab: 124572 kB\nPageTables: 6512 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 572456 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,922066944}, {buffered_memory,361013248}, {free_memory,474406912}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{681237,15}}, {context_switches,{180292,0}}, {garbage_collection,{49931,227309540,0}}, {io,{{input,16777360},{output,11129683}}}, {reductions,{94719917,1120275}}, {run_queue,0}, {runtime,{11450,170}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304151,306287}}, {active_buckets,[]}, {memory, [{total,29664352}, {processes,9858280}, {processes_used,9836808}, {system,19806072}, {atom,784585}, {atom_used,765874}, {binary,209192}, {code,7944128}, {ets,1551856}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,657}, {memory_data,{2058928128,1068896256,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966828 kB\nBuffers: 252628 kB\nCached: 510996 kB\nSwapCached: 17896 kB\nActive: 678420 kB\nInactive: 229784 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966828 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 96 kB\nWriteback: 0 kB\nAnonPages: 140020 kB\nMapped: 20308 kB\nSlab: 111528 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523259904}, {buffered_memory,258691072}, {free_memory,990031872}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{654806,2}}, {context_switches,{115020,0}}, {garbage_collection,{62944,417360291,0}}, {io,{{input,9152845},{output,7155855}}}, {reductions,{92114525,123397}}, {run_queue,0}, {runtime,{27090,20}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.855.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.859.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.859.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.861.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.861.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.864.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.430.0> 2011-04-08 16:09:22 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:09:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.864.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.866.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.866.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.869.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.869.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.871.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:09:38 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:09:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.871.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.874.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.874.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.876.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.876.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.879.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:09:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.879.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:09:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.881.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:09:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:09:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.881.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.884.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.884.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.886.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:10:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304211,64373}}, {active_buckets,[]}, {memory, [{total,15561072}, {processes,5036828}, {processes_used,5024948}, {system,10524244}, {atom,519117}, {atom_used,500490}, {binary,208008}, {code,4062708}, {ets,817876}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,670}, {memory_data,{2075373568,1765490688,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302620 kB\nBuffers: 246344 kB\nCached: 1252084 kB\nSwapCached: 0 kB\nActive: 737908 kB\nInactive: 887596 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300984 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 88 kB\nWriteback: 0 kB\nAnonPages: 127072 kB\nMapped: 28212 kB\nSlab: 85384 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282134016}, {buffered_memory,252256256}, {free_memory,309882880}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{661918,2}}, {context_switches,{122095,0}}, {garbage_collection,{79527,489685337,0}}, {io,{{input,9320087},{output,6113575}}}, {reductions,{107381607,130218}}, {run_queue,0}, {runtime,{28580,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304211,140908}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34878192}, {processes,13605856}, {processes_used,13583840}, {system,21272336}, {atom,828329}, {atom_used,823498}, {binary,165368}, {code,8669006}, {ets,2159464}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,749}, {memory_data,{2058928128,1582743552,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 464528 kB\nBuffers: 352552 kB\nCached: 900792 kB\nSwapCached: 17552 kB\nActive: 765020 kB\nInactive: 631524 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 464528 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 396 kB\nWriteback: 0 kB\nAnonPages: 138916 kB\nMapped: 17612 kB\nSlab: 124632 kB\nPageTables: 6500 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 570916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,922411008}, {buffered_memory,361013248}, {free_memory,475676672}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{741236,14}}, {context_switches,{192096,0}}, {garbage_collection,{54155,245968692,0}}, {io,{{input,19484185},{output,14179946}}}, {reductions,{101030982,964753}}, {run_queue,0}, {runtime,{12430,140}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304211,306632}}, {active_buckets,[]}, {memory, [{total,29784000}, {processes,9945872}, {processes_used,9924400}, {system,19838128}, {atom,784585}, {atom_used,765874}, {binary,207672}, {code,7944128}, {ets,1583976}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,717}, {memory_data,{2058928128,1068896256,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966828 kB\nBuffers: 252628 kB\nCached: 511112 kB\nSwapCached: 17896 kB\nActive: 678416 kB\nInactive: 229796 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966828 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 212 kB\nWriteback: 0 kB\nAnonPages: 139912 kB\nMapped: 20308 kB\nSlab: 111524 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523378688}, {buffered_memory,258691072}, {free_memory,990031872}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{714806,2}}, {context_switches,{119925,0}}, {garbage_collection,{63619,419926648,0}}, {io,{{input,9409035},{output,7312875}}}, {reductions,{92910174,170313}}, {run_queue,0}, {runtime,{27250,40}}]}]}] INFO REPORT <0.767.0> 2011-04-08 16:10:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.886.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.890.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.890.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.892.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.892.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.895.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.895.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.897.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:10:29 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:10:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.897.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.900.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.900.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.902.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.902.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.905.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.905.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.907.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:10:48 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:10:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.907.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.910.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:10:55 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:10:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:10:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.910.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:10:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.912.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:10:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:10:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.912.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.930.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.930.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.932.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:11:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304271,67005}}, {active_buckets,[]}, {memory, [{total,15357760}, {processes,4811060}, {processes_used,4799180}, {system,10546700}, {atom,519117}, {atom_used,500490}, {binary,210480}, {code,4062708}, {ets,836532}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,730}, {memory_data,{2075373568,1765617664,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302868 kB\nBuffers: 246344 kB\nCached: 1251776 kB\nSwapCached: 0 kB\nActive: 737848 kB\nInactive: 887372 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 301232 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 127076 kB\nMapped: 28212 kB\nSlab: 85388 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1281818624}, {buffered_memory,252256256}, {free_memory,310136832}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{721917,1}}, {context_switches,{127345,0}}, {garbage_collection,{80560,492464659,0}}, {io,{{input,9840869},{output,6395994}}}, {reductions,{108220978,194169}}, {run_queue,0}, {runtime,{28680,30}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304271,140614}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34942760}, {processes,13530360}, {processes_used,13508344}, {system,21412400}, {atom,828329}, {atom_used,823498}, {binary,150680}, {code,8669006}, {ets,2312568}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,809}, {memory_data,{2058928128,1583640576,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 463280 kB\nBuffers: 352552 kB\nCached: 901604 kB\nSwapCached: 17552 kB\nActive: 765548 kB\nInactive: 632068 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 463280 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 532 kB\nWriteback: 0 kB\nAnonPages: 139212 kB\nMapped: 17648 kB\nSlab: 124600 kB\nPageTables: 6596 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 570920 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,923242496}, {buffered_memory,361013248}, {free_memory,474398720}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{801233,11}}, {context_switches,{202640,0}}, {garbage_collection,{57700,262815800,0}}, {io,{{input,19716431},{output,14998995}}}, {reductions,{107052745,979735}}, {run_queue,0}, {runtime,{13320,130}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304271,307395}}, {active_buckets,[]}, {memory, [{total,29837992}, {processes,9961656}, {processes_used,9940184}, {system,19876336}, {atom,784585}, {atom_used,765874}, {binary,210568}, {code,7944128}, {ets,1616968}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,777}, {memory_data,{2058928128,1068896256,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966828 kB\nBuffers: 252628 kB\nCached: 510820 kB\nSwapCached: 17896 kB\nActive: 678528 kB\nInactive: 229680 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966828 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 192 kB\nWriteback: 0 kB\nAnonPages: 140204 kB\nMapped: 20308 kB\nSlab: 111516 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523079680}, {buffered_memory,258691072}, {free_memory,990031872}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{774806,2}}, {context_switches,{125095,0}}, {garbage_collection,{64361,422461006,0}}, {io,{{input,9926921},{output,7595072}}}, {reductions,{93701706,120643}}, {run_queue,0}, {runtime,{27420,30}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.932.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.936.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.936.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.938.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.938.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.941.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.941.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.943.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.943.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.946.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.946.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.948.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.948.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.951.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.951.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.953.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.953.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.956.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:11:54 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:11:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:11:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.956.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:11:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.958.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:11:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:11:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.958.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.961.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.961.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.963.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:12:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304331,68438}}, {active_buckets,[]}, {memory, [{total,15421368}, {processes,4858780}, {processes_used,4846900}, {system,10562588}, {atom,519117}, {atom_used,500490}, {binary,209040}, {code,4062708}, {ets,853012}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,790}, {memory_data,{2075373568,1765236736,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302744 kB\nBuffers: 246344 kB\nCached: 1251900 kB\nSwapCached: 0 kB\nActive: 737904 kB\nInactive: 887440 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 301108 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 152 kB\nWriteback: 0 kB\nAnonPages: 127096 kB\nMapped: 28212 kB\nSlab: 85376 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1281945600}, {buffered_memory,252256256}, {free_memory,310009856}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{781943,2}}, {context_switches,{132185,0}}, {garbage_collection,{81410,494930100,0}}, {io,{{input,10082992},{output,6553088}}}, {reductions,{108964202,126293}}, {run_queue,0}, {runtime,{28780,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304331,142120}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35006448}, {processes,13458576}, {processes_used,13436560}, {system,21547872}, {atom,828329}, {atom_used,823498}, {binary,196984}, {code,8669006}, {ets,2399976}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,869}, {memory_data,{2058928128,1585037312,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 462164 kB\nBuffers: 352552 kB\nCached: 902384 kB\nSwapCached: 17552 kB\nActive: 766156 kB\nInactive: 632616 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 462164 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 139588 kB\nMapped: 17648 kB\nSlab: 124612 kB\nPageTables: 6596 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 572096 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,924041216}, {buffered_memory,361013248}, {free_memory,473255936}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{861236,14}}, {context_switches,{213170,0}}, {garbage_collection,{61280,279588426,0}}, {io,{{input,19963440},{output,15835346}}}, {reductions,{113050203,956828}}, {run_queue,0}, {runtime,{14210,150}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304331,307500}}, {active_buckets,[]}, {memory, [{total,29916208}, {processes,10010888}, {processes_used,9989416}, {system,19905320}, {atom,784585}, {atom_used,765874}, {binary,206200}, {code,7944128}, {ets,1648864}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,837}, {memory_data,{2058928128,1068896256,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966704 kB\nBuffers: 252628 kB\nCached: 510932 kB\nSwapCached: 17896 kB\nActive: 678596 kB\nInactive: 229676 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966704 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 88 kB\nWriteback: 0 kB\nAnonPages: 140152 kB\nMapped: 20308 kB\nSlab: 111520 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523194368}, {buffered_memory,258691072}, {free_memory,989904896}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{834806,2}}, {context_switches,{129929,0}}, {garbage_collection,{65006,424866084,0}}, {io,{{input,10170159},{output,7752349}}}, {reductions,{94442511,121625}}, {run_queue,0}, {runtime,{27570,30}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.963.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.967.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.967.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.969.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:12:20 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:12:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.969.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.972.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.972.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.974.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.974.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.977.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.977.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.979.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.979.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.982.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.982.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.984.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.984.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.987.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:12:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.987.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:12:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.989.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:12:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:12:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.989.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.992.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.992.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.994.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:13:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304391,65048}}, {active_buckets,[]}, {memory, [{total,15386920}, {processes,4847396}, {processes_used,4835516}, {system,10539524}, {atom,519117}, {atom_used,500490}, {binary,207360}, {code,4062708}, {ets,830700}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,850}, {memory_data,{2075373568,1765363712,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302744 kB\nBuffers: 246344 kB\nCached: 1252012 kB\nSwapCached: 0 kB\nActive: 737976 kB\nInactive: 887496 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 301108 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 72 kB\nWriteback: 0 kB\nAnonPages: 127112 kB\nMapped: 28212 kB\nSlab: 85320 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518932 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282060288}, {buffered_memory,252256256}, {free_memory,310009856}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{841916,1}}, {context_switches,{137050,0}}, {garbage_collection,{82262,497479098,0}}, {io,{{input,10324960},{output,6710483}}}, {reductions,{109724729,149597}}, {run_queue,0}, {runtime,{28870,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304391,158774}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34865080}, {processes,13189880}, {processes_used,13167864}, {system,21675200}, {atom,828329}, {atom_used,823498}, {binary,165424}, {code,8669006}, {ets,2555664}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,929}, {memory_data,{2058928128,1594585088,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 427092 kB\nBuffers: 352552 kB\nCached: 899692 kB\nSwapCached: 17552 kB\nActive: 765700 kB\nInactive: 630448 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 427092 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 444 kB\nWriteback: 0 kB\nAnonPages: 139648 kB\nMapped: 17648 kB\nSlab: 124764 kB\nPageTables: 6600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,921284608}, {buffered_memory,361013248}, {free_memory,437342208}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{921236,14}}, {context_switches,{224405,0}}, {garbage_collection,{65234,296607171,0}}, {io,{{input,22664807},{output,17997711}}}, {reductions,{119099853,955664}}, {run_queue,0}, {runtime,{15160,140}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304391,308863}}, {active_buckets,[]}, {memory, [{total,29858624}, {processes,9984200}, {processes_used,9962728}, {system,19874424}, {atom,784585}, {atom_used,765874}, {binary,206760}, {code,7944128}, {ets,1615760}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,897}, {memory_data,{2058928128,1068769280,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966828 kB\nBuffers: 252628 kB\nCached: 511048 kB\nSwapCached: 17896 kB\nActive: 678628 kB\nInactive: 229680 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966828 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 192 kB\nWriteback: 0 kB\nAnonPages: 140072 kB\nMapped: 20308 kB\nSlab: 111516 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523313152}, {buffered_memory,258691072}, {free_memory,990031872}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{894808,4}}, {context_switches,{134804,0}}, {garbage_collection,{65670,427401545,0}}, {io,{{input,10426645},{output,7908506}}}, {reductions,{95205347,146786}}, {run_queue,0}, {runtime,{27700,20}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.994.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.998.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:13:15 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:13:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.998.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1000.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1000.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1003.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1003.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1005.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1005.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1008.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:13:36 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:13:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1008.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1010.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1010.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1013.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1013.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1015.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1015.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1018.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:13:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1018.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:13:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1020.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:13:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:13:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1020.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1036.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1036.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1038.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:14:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304451,64546}}, {active_buckets,[]}, {memory, [{total,15408720}, {processes,4848908}, {processes_used,4837028}, {system,10559812}, {atom,519117}, {atom_used,500490}, {binary,208288}, {code,4062708}, {ets,848628}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,910}, {memory_data,{2075373568,1765490688,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302620 kB\nBuffers: 246352 kB\nCached: 1252140 kB\nSwapCached: 0 kB\nActive: 737988 kB\nInactive: 887604 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300984 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 140 kB\nWriteback: 28 kB\nAnonPages: 127116 kB\nMapped: 28212 kB\nSlab: 85356 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518932 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282191360}, {buffered_memory,252264448}, {free_memory,309882880}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{901917,2}}, {context_switches,{142248,0}}, {garbage_collection,{83371,500075913,0}}, {io,{{input,10828219},{output,7089665}}}, {reductions,{110508433,163726}}, {run_queue,0}, {runtime,{29000,40}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304451,142689}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34509064}, {processes,12809880}, {processes_used,12787864}, {system,21699184}, {atom,828329}, {atom_used,823498}, {binary,169400}, {code,8669006}, {ets,2573912}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,989}, {memory_data,{2058928128,1583386624,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 463884 kB\nBuffers: 352552 kB\nCached: 900464 kB\nSwapCached: 17552 kB\nActive: 766168 kB\nInactive: 631000 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 463884 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 139892 kB\nMapped: 17648 kB\nSlab: 124644 kB\nPageTables: 6600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,922075136}, {buffered_memory,361013248}, {free_memory,475017216}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{981231,9}}, {context_switches,{234907,0}}, {garbage_collection,{69033,312661411,0}}, {io,{{input,22941814},{output,18827912}}}, {reductions,{124961297,935116}}, {run_queue,0}, {runtime,{16010,130}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304451,308275}}, {active_buckets,[]}, {memory, [{total,29912448}, {processes,9999976}, {processes_used,9978504}, {system,19912472}, {atom,784585}, {atom_used,765874}, {binary,208720}, {code,7944128}, {ets,1649384}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,957}, {memory_data,{2058928128,1068896256,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966704 kB\nBuffers: 252628 kB\nCached: 511144 kB\nSwapCached: 17896 kB\nActive: 678452 kB\nInactive: 229860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966704 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 188 kB\nWriteback: 0 kB\nAnonPages: 139980 kB\nMapped: 20308 kB\nSlab: 111524 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523411456}, {buffered_memory,258691072}, {free_memory,989904896}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{954806,2}}, {context_switches,{139969,0}}, {garbage_collection,{66507,429871899,0}}, {io,{{input,10914650},{output,8290670}}}, {reductions,{95972439,121674}}, {run_queue,0}, {runtime,{27870,20}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1038.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1042.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1042.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1044.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1044.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1047.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:14:24 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:14:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1047.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1049.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1049.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1052.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1052.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1054.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1054.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1057.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1057.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1059.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1059.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1062.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:14:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1062.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:14:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1064.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:14:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:14:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1064.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1067.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:15:02 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:15:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1067.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1069.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:15:11 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.458.0> 2011-04-08 16:15:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304511,66217}}, {active_buckets,[]}, {memory, [{total,15327816}, {processes,4806148}, {processes_used,4794268}, {system,10521668}, {atom,519117}, {atom_used,500490}, {binary,210392}, {code,4062708}, {ets,807436}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,970}, {memory_data,{2075373568,1765617664,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302496 kB\nBuffers: 246352 kB\nCached: 1252228 kB\nSwapCached: 0 kB\nActive: 738040 kB\nInactive: 887660 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300860 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 168 kB\nWriteback: 0 kB\nAnonPages: 127116 kB\nMapped: 28212 kB\nSlab: 85328 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518932 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282281472}, {buffered_memory,252264448}, {free_memory,309755904}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{961916,1}}, {context_switches,{147124,0}}, {garbage_collection,{84224,502544947,0}}, {io,{{input,11070333},{output,7244459}}}, {reductions,{111242169,123945}}, {run_queue,0}, {runtime,{29100,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304511,143882}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35012800}, {processes,13178952}, {processes_used,13156936}, {system,21833848}, {atom,828329}, {atom_used,823498}, {binary,149208}, {code,8669006}, {ets,2727200}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1049}, {memory_data,{2058928128,1584517120,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 462984 kB\nBuffers: 352552 kB\nCached: 901240 kB\nSwapCached: 17552 kB\nActive: 766344 kB\nInactive: 631564 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 462984 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 196 kB\nWriteback: 0 kB\nAnonPages: 139848 kB\nMapped: 17648 kB\nSlab: 124660 kB\nPageTables: 6596 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,922869760}, {buffered_memory,361013248}, {free_memory,474095616}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1041235,12}}, {context_switches,{245360,0}}, {garbage_collection,{72841,328675698,0}}, {io,{{input,23174163},{output,19670515}}}, {reductions,{130822260,943193}}, {run_queue,0}, {runtime,{16830,130}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304511,308737}}, {active_buckets,[]}, {memory, [{total,29699480}, {processes,9886848}, {processes_used,9865376}, {system,19812632}, {atom,784585}, {atom_used,765874}, {binary,202824}, {code,7944128}, {ets,1553792}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1017}, {memory_data,{2058928128,1069023232,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966572 kB\nBuffers: 252628 kB\nCached: 511276 kB\nSwapCached: 17896 kB\nActive: 678616 kB\nInactive: 229860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966572 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 172 kB\nWriteback: 0 kB\nAnonPages: 140008 kB\nMapped: 20312 kB\nSlab: 111540 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 551836 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523546624}, {buffered_memory,258691072}, {free_memory,989769728}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1014806,2}}, {context_switches,{144867,0}}, {garbage_collection,{67177,432455663,0}}, {io,{{input,11200810},{output,8446805}}}, {reductions,{96746314,151846}}, {run_queue,0}, {runtime,{28020,20}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1069.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1073.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:15:16 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1073.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1075.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1075.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1078.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1078.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1080.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1080.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1083.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1083.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1085.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1085.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1088.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1088.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1090.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:51 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1090.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1093.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:15:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:15:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1093.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:15:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1095.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.430.0> 2011-04-08 16:15:57 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:15:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:15:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1095.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1098.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:16:03 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:16:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1098.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1100.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:16:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304571,65921}}, {active_buckets,[]}, {memory, [{total,15383944}, {processes,4845988}, {processes_used,4834108}, {system,10537956}, {atom,519117}, {atom_used,500490}, {binary,209168}, {code,4062708}, {ets,824116}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,1030}, {memory_data,{2075373568,1765744640,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302372 kB\nBuffers: 246352 kB\nCached: 1252348 kB\nSwapCached: 0 kB\nActive: 738100 kB\nInactive: 887716 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300736 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 92 kB\nWriteback: 0 kB\nAnonPages: 127120 kB\nMapped: 28212 kB\nSlab: 85332 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518932 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282404352}, {buffered_memory,252264448}, {free_memory,309628928}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{1021917,2}}, {context_switches,{152075,0}}, {garbage_collection,{85108,505162678,0}}, {io,{{input,11357521},{output,7403665}}}, {reductions,{112003651,148620}}, {run_queue,0}, {runtime,{29210,10}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304571,144090}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35180432}, {processes,13188736}, {processes_used,13166720}, {system,21991696}, {atom,828329}, {atom_used,823498}, {binary,147984}, {code,8669006}, {ets,2883208}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1109}, {memory_data,{2058928128,1587236864,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 453816 kB\nBuffers: 352560 kB\nCached: 908140 kB\nSwapCached: 17552 kB\nActive: 770780 kB\nInactive: 635456 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 453816 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 7028 kB\nWriteback: 0 kB\nAnonPages: 141276 kB\nMapped: 17652 kB\nSlab: 125100 kB\nPageTables: 6848 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 575952 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,929935360}, {buffered_memory,361021440}, {free_memory,464707584}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1101236,14}}, {context_switches,{256655,0}}, {garbage_collection,{76821,345276788,0}}, {io,{{input,25891849},{output,22752728}}}, {reductions,{136845746,954067}}, {run_queue,0}, {runtime,{17770,140}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304571,309158}}, {active_buckets,[]}, {memory, [{total,29855864}, {processes,10002992}, {processes_used,9981520}, {system,19852872}, {atom,784585}, {atom_used,765874}, {binary,210176}, {code,7944128}, {ets,1585224}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1077}, {memory_data,{2058928128,1069150208,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966332 kB\nBuffers: 252632 kB\nCached: 511396 kB\nSwapCached: 17896 kB\nActive: 678788 kB\nInactive: 229860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966332 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 1968 kB\nWriteback: 0 kB\nAnonPages: 140060 kB\nMapped: 20312 kB\nSlab: 111560 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 551840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523669504}, {buffered_memory,258695168}, {free_memory,989523968}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1074806,2}}, {context_switches,{149740,0}}, {garbage_collection,{67892,434900141,0}}, {io,{{input,11457595},{output,8604849}}}, {reductions,{97532561,166245}}, {run_queue,0}, {runtime,{28170,30}}]}]}] SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1100.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1104.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1104.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1106.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:16:21 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.767.0> 2011-04-08 16:16:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1106.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1109.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1109.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1111.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1111.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1114.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1114.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1116.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:42 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1116.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:42 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1119.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:42 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:43 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:44 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:45 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:46 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:47 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1119.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:47 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1121.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:47 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:48 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:49 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:50 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:52 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:52 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1121.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:52 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1124.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:53 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:54 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:55 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:56 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:57 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:16:57 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1124.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:16:57 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1126.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:16:58 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:16:59 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:00 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:01 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:02 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:02 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1126.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:02 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1144.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:03 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:04 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:05 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:06 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:07 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:07 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1144.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:07 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1146.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:08 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:09 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:10 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:11 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.458.0> 2011-04-08 16:17:12 =============================================================================== ns_1@10.1.5.178:<0.458.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304631,64543}}, {active_buckets,[]}, {memory, [{total,15382712}, {processes,4820692}, {processes_used,4808812}, {system,10562020}, {atom,519117}, {atom_used,500490}, {binary,213984}, {code,4062708}, {ets,842084}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,1090}, {memory_data,{2075373568,1765871616,{<0.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302620 kB\nBuffers: 246352 kB\nCached: 1252040 kB\nSwapCached: 0 kB\nActive: 738024 kB\nInactive: 887496 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300984 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 116 kB\nWriteback: 0 kB\nAnonPages: 127120 kB\nMapped: 28212 kB\nSlab: 85316 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518932 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282088960}, {buffered_memory,252264448}, {free_memory,309882880}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{1081915,1}}, {context_switches,{157354,0}}, {garbage_collection,{86163,507905207,0}}, {io,{{input,11863878},{output,7735227}}}, {reductions,{112843114,172896}}, {run_queue,0}, {runtime,{29350,40}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304631,143545}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35360272}, {processes,13202632}, {processes_used,13180616}, {system,22157640}, {atom,828329}, {atom_used,823498}, {binary,159632}, {code,8669006}, {ets,3035936}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1169}, {memory_data,{2058928128,1594331136,{<10463.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 453320 kB\nBuffers: 352560 kB\nCached: 908916 kB\nSwapCached: 17552 kB\nActive: 770992 kB\nInactive: 635992 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 453320 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 540 kB\nWriteback: 0 kB\nAnonPages: 141252 kB\nMapped: 17652 kB\nSlab: 124940 kB\nPageTables: 6852 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 575260 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,930729984}, {buffered_memory,361021440}, {free_memory,464199680}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1161237,15}}, {context_switches,{267143,0}}, {garbage_collection,{80588,361337501,0}}, {io,{{input,26169123},{output,23582947}}}, {reductions,{142694424,927544}}, {run_queue,0}, {runtime,{18600,130}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304631,309453}}, {active_buckets,[]}, {memory, [{total,29852872}, {processes,9968176}, {processes_used,9946704}, {system,19884696}, {atom,784585}, {atom_used,765874}, {binary,206440}, {code,7944128}, {ets,1618528}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1137}, {memory_data,{2058928128,1069404160,{<10507.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966704 kB\nBuffers: 252636 kB\nCached: 511088 kB\nSwapCached: 17896 kB\nActive: 678536 kB\nInactive: 229732 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966704 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 208 kB\nWriteback: 0 kB\nAnonPages: 139984 kB\nMapped: 20312 kB\nSlab: 111544 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 551840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523354112}, {buffered_memory,258699264}, {free_memory,989904896}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1134806,2}}, {context_switches,{154911,0}}, {garbage_collection,{68654,437467316,0}}, {io,{{input,11962376},{output,8936400}}}, {reductions,{98326482,126846}}, {run_queue,0}, {runtime,{28360,30}}]}]}] INFO REPORT <0.767.0> 2011-04-08 16:17:12 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:12 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1146.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:12 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1150.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:13 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:14 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.430.0> 2011-04-08 16:17:14 =============================================================================== Pulling config from: 'ns_1@10.1.5.193' INFO REPORT <0.767.0> 2011-04-08 16:17:15 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:16 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:17 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:17 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1150.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:17 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1152.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:18 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:19 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:20 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:21 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:22 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:22 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1152.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:22 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1155.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:23 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:24 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:25 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:26 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:27 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:27 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1155.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:27 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1157.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:28 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:29 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:30 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:31 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:32 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:32 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1157.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:32 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1161.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:33 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:34 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:35 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:36 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:37 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.460.0> 2011-04-08 16:17:37 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1161.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.460.0> 2011-04-08 16:17:37 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1164.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.767.0> 2011-04-08 16:17:38 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:39 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:40 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.767.0> 2011-04-08 16:17:41 =============================================================================== ns_1@10.1.5.178:<0.767.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it.