per_node_diag = [{'ns_1@10.1.5.193', [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {manifest, ["bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz", "curl-7.21.1-w64_patched.tar.gz", "ep-engine_1.6.5.3_143_g366c22e-Linux.x86_64.tar.gz", "google-perftools-1.7","google-perftools-1.7.tar.gz", "libconflate_1.6.5_1_g8a42194-Linux.x86_64.tar.gz", "libevent-2.0.7-rc.tar.gz", "libmemcached-0.41_trond-norbye_mingw32-revno895.tar.gz", "libvbucket_1.6.4_1_g5cb588e-Linux.x86_64.tar.gz", "membase-cli_1.6.5_2_gb9c8a7d-Linux.x86_64.tar.gz", "memcached_1.4.4_430_g6235145-Linux.x86_64.tar.gz", "moxi_1.6.5.1_16_g8128746-Linux.x86_64.tar.gz", "ns_server_1.6.5.3_291_gba5fe27.tar.gz", "portsigar_-Linux.x86_64.tar.gz", "sigar_sigar_1.6.4_402_g7809620-Linux.x86_64.tar.gz", "vbucketmigrator_1.7.0r_4_g8428c29-Linux.x86_64.tar.gz", "wallace_1.7.0r-100-g85ee8f5-Linux.x86_64-201104060842"]}, {config, [{alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,'filtered-out'}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up, server_joined,server_left,bucket_created, bucket_deleted,bucket_auth_failed]}]}, {buckets, [{'_vclock',[{'ns_1@10.1.5.192',{11,63469523235}}]}, {configs, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers, ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178']}, {map, [['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}]}]}, {memory_quota,1570}, {nodes_wanted, [{'_vclock',[{'ns_1@10.1.5.192',{2,63469523039}}]}, 'ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193']}, {otp, [{'_vclock',[{'ns_1@10.1.5.193',{1,63469523027}}]}, {cookie,lulxzmudqfmlvwvs}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {rebalance_status, {none, <<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>}}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds, [{'_vclock',[{'ns_1@10.1.5.192',{1,63469522657}}]}, {creds, [{"Administrator",[{password,'filtered-out'}]}]}]}, {{node,'ns_1@10.1.5.178',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.178',membership},inactiveFailed}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.178',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.192',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.192',membership},active}, {{node,'ns_1@10.1.5.192',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.192',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.192',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.192',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.193',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.193',membership},inactiveFailed}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.193',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}]}, {basic_info, [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1160}, {memory_data,{2058928128,1069023232,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}, {processes, [{<0.0.0>, [{registered_name,init}, {status,waiting}, {initial_call,{otp_ring0,start,2}}, {backtrace, ["Program counter: 0x00002b8796585290 (init:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabfd7230 Return addr 0x0000000000816eb8 ()", "y(0) {state,[{'-root',[<<37 bytes>>]},{'-progname',[<<3 bytes>>]},{'-home',[<<12 bytes"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,84}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<0.6.0>,<0.7.0>,<0.3.0>]}, {memory,108976}, {message_queue_len,0}, {reductions,46449}, {trap_exit,true}]}, {<0.3.0>, [{registered_name,erl_prim_loader}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b87965e4d20 (erl_prim_loader:loop/3 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabd0a6f0 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) [\"/opt/membase/1.7.0r/bin/ns_server/ebin\",\"/opt/membase/1.7.0r/bin/ns_server/deps", "y(2) <0.2.0>", "y(3) {state,efile,[],none,#Port<0.2>,infinity,undefined,true,{prim_state,false,undefin", "y(4) infinity"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,224}]}, {heap_size,6765}, {total_heap_size,53133}, {links,[#Port<0.2>,<0.0.0>]}, {memory,425760}, {message_queue_len,0}, {reductions,936052}, {trap_exit,true}]}, {<0.6.0>, [{registered_name,error_logger}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1e240 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_log_mf_h,false,{state,\"/var/membase/logs\",10485760,10,606042,1,{file", "y(3) error_logger","y(4) <0.2.0>", "0x00002aaaaae1e270 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,38}]}, {heap_size,610}, {total_heap_size,1220}, {links,[<0.0.0>,<0.25.0>,#Port<0.981>]}, {memory,10600}, {message_queue_len,0}, {reductions,210832}, {trap_exit,true}]}, {<0.7.0>, [{registered_name,application_controller}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796642d50 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) infinity", "y(2) application_controller", "y(3) {state,[],[],[],[{ns_server,<0.52.0>},{mnesia,<0.70.0>},{os_mon,<0.41.0>},{sasl,<", "y(4) application_controller", "y(5) <0.2.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [<0.41.0>,<0.52.0>,<0.70.0>,<0.9.0>,<0.32.0>,<0.0.0>]}, {memory,263664}, {message_queue_len,0}, {reductions,30671}, {trap_exit,true}]}, {<0.9.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966492f0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.10.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_ser", "y(2) <0.7.0>", "0x00002b8796649310 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.7.0>,<0.10.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<0.10.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796639790 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) kernel","y(2) <0.11.0>", "y(3) <0.9.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.9.0>,<0.11.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,72}, {trap_exit,true}]}, {<0.11.0>, [{registered_name,kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadba358 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_sup},one_for_all,[{child,<0.58.0>,net_sup_dynamic,{erl_distr", "y(4) kernel_sup","y(5) <0.10.0>", "0x00002aaaaadba390 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<0.22.0>,<0.26.0>,<0.27.0>,<0.58.0>,<0.24.0>, <0.17.0>,<0.20.0>,<0.21.0>,<0.19.0>,<0.12.0>, <0.13.0>,<0.10.0>]}, {memory,12112}, {message_queue_len,0}, {reductions,1492}, {trap_exit,true}]}, {<0.12.0>, [{registered_name,rex}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac9fd9f0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) rpc", "y(3) {1,{<0.1194.0>,{<0.1193.0>,{#Ref<0.0.0.24792>,'ns_1@10.1.5.193'}},nil,nil}}", "y(4) rex","y(5) <0.11.0>", "0x00002aaaac9fda28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,196418}, {total_heap_size,196418}, {links,[<0.11.0>]}, {memory,1572248}, {message_queue_len,0}, {reductions,2917}, {trap_exit,true}]}, {<0.13.0>, [{registered_name,global_name_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0f8cc8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) global", "y(3) {state,true,['ns_1@10.1.5.192','ns_1@10.1.5.178'],['ns_1@10.1.5.178','ns_1@10.1.5", "y(4) global_name_server","y(5) <0.11.0>", "0x00002aaaab0f8d00 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.14.0>,<0.16.0>,<0.15.0>,<0.11.0>]}, {memory,16880}, {message_queue_len,0}, {reductions,2067}, {trap_exit,true}]}, {<0.14.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac82618 (global:loop_the_locker/1 + 1176)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae31268 Return addr 0x00002aaaaac82140 (global:init_the_locker/1 + 384)", "y(0) {multi,[],[],['ns_1@10.1.5.178','ns_1@10.1.5.192'],'ns_1@10.1.5.193',false,false}", "y(1) infinity", "0x00002aaaaae31280 Return addr 0x0000000000816eb8 ()"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,565}, {trap_exit,true}]}, {<0.15.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a770 (global:collect_deletions/2 + 152)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb5118 Return addr 0x00002aaaaac8a670 (global:loop_the_deleter/1 + 72)", "y(0) infinity","y(1) []", "y(2) <0.13.0>", "0x00002aaaaadb5138 Return addr 0x0000000000816eb8 ()", "y(0) <0.13.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.13.0>]}, {memory,6688}, {message_queue_len,0}, {reductions,231}, {trap_exit,false}]}, {<0.16.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a9b0 (global:loop_the_registrar/0 + 24)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae53028 Return addr 0x0000000000816eb8 ()", "y(0) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,248}, {trap_exit,false}]}, {<0.17.0>, [{registered_name,inet_db}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796675e20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) inet_db", "y(3) {state,inet_db,inet_cache,inet_hosts_byname,inet_hosts_byaddr,inet_hosts_file_byn", "y(4) inet_db","y(5) <0.11.0>", "0x00002b8796675e58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<0.11.0>]}, {memory,26312}, {message_queue_len,0}, {reductions,1457}, {trap_exit,true}]}, {<0.19.0>, [{registered_name,global_group}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879664a5f8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) global_group", "y(3) {state,no_conf,true,[],[],[],[],[],'nonode@nohost',[],normal,normal}", "y(4) global_group","y(5) <0.11.0>", "0x00002b879664a630 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,84}, {trap_exit,true}]}, {<0.20.0>, [{registered_name,file_server_2}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966583b8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) file_server","y(3) #Port<0.223>", "y(4) file_server_2","y(5) <0.11.0>", "0x00002b87966583f0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,311}]}, {heap_size,2584}, {total_heap_size,13530}, {links,[#Port<0.223>,<0.11.0>]}, {memory,109040}, {message_queue_len,0}, {reductions,603408}, {trap_exit,true}]}, {<0.21.0>, [{registered_name,code_server}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaad54b40 (code_server:loop/1 + 128)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0d3188 Return addr 0x0000000000816eb8 ()", "y(0) {state,<0.11.0>,\"/opt/membase/1.7.0r/erlang/lib/erlang\",[\"/opt/membase/1.7.0r/bin", "y(1) <0.11.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,148}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[<0.11.0>]}, {memory,263360}, {message_queue_len,0}, {reductions,216171}, {trap_exit,true}]}, {<0.22.0>, [{registered_name,standard_error_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666c3d0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,standard_error,<0.23.0>,<0.23.0>,{local,standard_error_sup}}", "y(4) standard_error_sup","y(5) <0.11.0>", "0x00002b879666c408 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.11.0>,<0.23.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<0.23.0>, [{registered_name,standard_error}, {status,waiting}, {initial_call,{standard_error,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad203a0 (standard_error:server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666d2a8 Return addr 0x0000000000816eb8 ()", "y(0) #Port<0.506>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.22.0>,#Port<0.506>]}, {memory,2664}, {message_queue_len,0}, {reductions,7}, {trap_exit,true}]}, {<0.24.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb8478 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,user_sup,<0.25.0>,<0.25.0>,{<0.24.0>,user_sup}}", "y(4) <0.24.0>","y(5) <0.11.0>", "0x00002aaaaadb84b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<0.11.0>,<0.25.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,71}, {trap_exit,true}]}, {<0.25.0>, [{registered_name,user}, {status,waiting}, {initial_call,{user,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad765e0 (user:server_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc76e8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) {[],[]}", "y(3) #Port<0.523>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<0.6.0>,<0.24.0>,#Port<0.523>]}, {memory,109080}, {message_queue_len,0}, {reductions,27509}, {trap_exit,true}]}, {<0.26.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666cb20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) kernel_config","y(3) []", "y(4) <0.26.0>","y(5) <0.11.0>", "0x00002b879666cb58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,176}, {trap_exit,true}]}, {<0.27.0>, [{registered_name,kernel_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae97bb8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_safe_sup},one_for_one,[{child,<0.92.0>,dets,{dets_server,sta", "y(4) kernel_safe_sup","y(5) <0.11.0>", "0x00002aaaaae97bf0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<0.87.0>,<0.91.0>,<0.92.0>,<0.88.0>,<0.50.0>, <0.56.0>,<0.11.0>]}, {memory,7032}, {message_queue_len,0}, {reductions,439}, {trap_exit,true}]}, {<0.32.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879664e240 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.33.0>,{appl_data,sasl,[sasl_sup,alarm_handler,overload,release_handler]", "y(2) <0.7.0>", "0x00002b879664e260 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.7.0>,<0.33.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<0.33.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796646df0 Return addr 0x0000000000816eb8 ()", "y(0) {state,tty,undefined}","y(1) sasl", "y(2) <0.34.0>","y(3) <0.32.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.32.0>,<0.34.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,70}, {trap_exit,true}]}, {<0.34.0>, [{registered_name,sasl_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796647c70 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_sup},one_for_one,[{child,<0.38.0>,release_handler,{release_han", "y(4) sasl_sup","y(5) <0.33.0>", "0x00002b8796647ca8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.35.0>,<0.38.0>,<0.33.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<0.35.0>, [{registered_name,sasl_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796633f50 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_safe_sup},one_for_one,[{child,<0.37.0>,overload,{overload,star", "y(4) sasl_safe_sup","y(5) <0.34.0>", "0x00002b8796633f88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.36.0>,<0.37.0>,<0.34.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,174}, {trap_exit,true}]}, {<0.36.0>, [{registered_name,alarm_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666d9c8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,alarm_handler,false,[],false}]", "y(3) alarm_handler","y(4) <0.35.0>", "0x00002b879666d9f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,28}, {trap_exit,true}]}, {<0.37.0>, [{registered_name,overload}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666e860 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) overload", "y(3) {state,0,0,8.000000e-01,556,1.000000e-01,{0,0},clear}", "y(4) overload","y(5) <0.35.0>", "0x00002b879666e898 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,39}, {trap_exit,false}]}, {<0.38.0>, [{registered_name,release_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea2440 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) release_handler", "y(3) {state,[],\"/opt/membase/1.7.0r/erlang/lib/erlang\",\"/opt/membase/1.7.0r/erlang/lib", "y(4) release_handler","y(5) <0.34.0>", "0x00002aaaaaea2478 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,4181}, {links,[<0.34.0>]}, {memory,34208}, {message_queue_len,0}, {reductions,2237}, {trap_exit,false}]}, {<0.41.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879666e128 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.42.0>,{appl_data,os_mon,[os_mon_sup,os_mon_sysinfo,disksup,memsup,cpu_s", "y(2) <0.7.0>", "0x00002b879666e148 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.7.0>,<0.42.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<0.42.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796645390 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) os_mon","y(2) <0.43.0>", "y(3) <0.41.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.41.0>,<0.43.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<0.43.0>, [{registered_name,os_mon_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879664d808 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,os_mon_sup},one_for_one,[{child,<0.47.0>,cpu_sup,{cpu_sup,start_lin", "y(4) os_mon_sup","y(5) <0.42.0>", "0x00002b879664d840 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.44.0>,<0.45.0>,<0.47.0>,<0.42.0>]}, {memory,6912}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<0.44.0>, [{registered_name,disksup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966532f0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) disksup", "y(3) {state,80,60000,{unix,linux},[{\"/\",13140720,39},{\"/boot\",101086,13},{\"/dev/shm\",1", "y(4) disksup","y(5) <0.43.0>", "0x00002b8796653328 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,33}]}, {heap_size,2584}, {total_heap_size,5168}, {links,[<0.43.0>,#Port<0.879>]}, {memory,42144}, {message_queue_len,0}, {reductions,28317}, {trap_exit,true}]}, {<0.45.0>, [{registered_name,memsup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966aabe0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) memsup", "y(3) {state,{unix,linux},true,{1069023232,2058928128},{<0.427.0>,1572784},false,60000,", "y(4) memsup","y(5) <0.43.0>", "0x00002b87966aac18 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,48}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<0.43.0>,<0.46.0>]}, {memory,26352}, {message_queue_len,0}, {reductions,45782}, {trap_exit,true}]}, {<0.46.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf49870 (memsup:port_idle/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae3bb20 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.908>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,34}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.45.0>,#Port<0.908>]}, {memory,8592}, {message_queue_len,0}, {reductions,12756}, {trap_exit,true}]}, {<0.47.0>, [{registered_name,cpu_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadce1f8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) cpu_sup", "y(3) {state,<0.48.0>,{unix,linux}}", "y(4) cpu_sup","y(5) <0.43.0>", "0x00002aaaaadce230 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.43.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,33}, {trap_exit,true}]}, {<0.48.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3a528 (cpu_sup:measurement_server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcf510 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []","y(5) []", "y(6) []","y(7) []", "y(8) {internal,<0.49.0>,[],{unix,linux}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.49.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,11}, {trap_exit,true}]}, {<0.49.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3b700 (cpu_sup:port_server_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796638480 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) 6000", "y(2) #Port<0.946>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.48.0>,#Port<0.946>]}, {memory,8592}, {message_queue_len,0}, {reductions,299}, {trap_exit,false}]}, {<0.50.0>, [{registered_name,timer_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae28f78 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) 219","y(2) timer", "y(3) []","y(4) timer_server", "y(5) <0.27.0>", "0x00002aaaaae28fb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1878}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<0.460.0>,<0.480.0>,<0.1191.0>,<0.467.0>,<0.428.0>, <0.446.0>,<0.459.0>,<0.432.0>,<0.78.0>,<0.82.0>, <0.27.0>]}, {memory,12072}, {message_queue_len,0}, {reductions,1261952}, {trap_exit,true}]}, {<0.52.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02ebd8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.53.0>,{appl_data,ns_server,[ns_server_sup,ns_config,ns_config_sup,ns_co", "y(2) <0.7.0>", "0x00002aaaab02ebf8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.7.0>,<0.53.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,48}, {trap_exit,true}]}, {<0.53.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadca470 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) ns_server", "y(2) <0.54.0>","y(3) <0.52.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.52.0>,<0.54.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,50}, {trap_exit,true}]}, {<0.54.0>, [{registered_name,ns_server_cluster_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea7508 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_cluster_sup},one_for_one,[{child,<0.427.0>,ns_server_sup,", "y(4) ns_server_cluster_sup","y(5) <0.53.0>", "0x00002aaaaaea7540 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,2584}, {total_heap_size,5168}, {links, [<0.65.0>,<0.117.0>,<0.427.0>,<0.55.0>,<0.64.0>, <0.53.0>]}, {memory,42304}, {message_queue_len,0}, {reductions,1906}, {trap_exit,true}]}, {<0.55.0>, [{registered_name,dist_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796647520 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dist_manager", "y(3) {state,true,\"10.1.5.193\"}", "y(4) dist_manager","y(5) <0.54.0>", "0x00002b8796647558 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.54.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,912}, {trap_exit,false}]}, {<0.56.0>, [{registered_name,inet_gethost_native_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcd660 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,inet_gethost_native,<0.57.0>,<0.57.0>,{local,inet_gethost_native_sup}}", "y(4) inet_gethost_native_sup", "y(5) <0.27.0>", "0x00002aaaaadcd698 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.27.0>,<0.57.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,41}, {trap_exit,true}]}, {<0.57.0>, [{registered_name,inet_gethost_native}, {status,waiting}, {initial_call,{inet_gethost_native,server_init,2}}, {backtrace, ["Program counter: 0x00002aaaaacc9bd8 (inet_gethost_native:main_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaeaabc8 Return addr 0x0000000000816eb8 ()", "y(0) {state,#Port<0.1012>,8000,8209,12306,<0.56.0>,4,{statistics,0,0,0,0,0,0,0,0}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,296}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.56.0>,#Port<0.1012>]}, {memory,6832}, {message_queue_len,0}, {reductions,12870}, {trap_exit,true}]}, {<0.58.0>, [{registered_name,net_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796688e28 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,net_sup},one_for_all,[{child,<0.61.0>,net_kernel,{net_kernel,start_", "y(4) net_sup","y(5) <0.11.0>", "0x00002b8796688e60 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.59.0>,<0.60.0>,<0.61.0>,<0.11.0>]}, {memory,8776}, {message_queue_len,0}, {reductions,252}, {trap_exit,true}]}, {<0.59.0>, [{registered_name,erl_epmd}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796689cc8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) erl_epmd", "y(3) {state,#Port<0.1048>,21100,ns_1}", "y(4) erl_epmd","y(5) <0.58.0>", "0x00002b8796689d00 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.58.0>,#Port<0.1048>]}, {memory,2664}, {message_queue_len,0}, {reductions,127}, {trap_exit,false}]}, {<0.60.0>, [{registered_name,auth}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1541d0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) auth", "y(3) {state,lulxzmudqfmlvwvs,16403}", "y(4) auth","y(5) <0.58.0>", "0x00002aaaab154208 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.58.0>]}, {memory,6792}, {message_queue_len,0}, {reductions,383}, {trap_exit,true}]}, {<0.61.0>, [{registered_name,net_kernel}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796658f88 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) net_kernel", "y(3) {state,'ns_1@10.1.5.193','ns_1@10.1.5.193',longnames,{tick,<0.63.0>,15000},7000,s", "y(4) net_kernel","y(5) <0.58.0>", "0x00002b8796658fc0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,18}]}, {heap_size,377}, {total_heap_size,754}, {links, [<0.63.0>,<0.422.0>,<0.500.0>,<0.58.0>,<0.62.0>, #Port<0.1046>]}, {memory,7024}, {message_queue_len,0}, {reductions,2218}, {trap_exit,true}]}, {<0.62.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,accept_loop,2}}, {backtrace, ["Program counter: 0x00002b87965b7ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc9cf8 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 79","y(1) #Port<0.1046>", "0x00002aaaaadc9d10 Return addr 0x00002aaaaaf84540 (inet_tcp_dist:accept_loop/2 + 96)", "y(0) []", "0x00002aaaaadc9d20 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.1046>", "y(2) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.61.0>]}, {memory,5608}, {message_queue_len,0}, {reductions,525}, {trap_exit,false}]}, {<0.63.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{net_kernel,ticker,2}}, {backtrace, ["Program counter: 0x00002aaaaad358c8 (net_kernel:ticker_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796638bd8 Return addr 0x0000000000816eb8 ()", "y(0) 15000","y(1) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.61.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,157}, {trap_exit,false}]}, {<0.64.0>, [{registered_name,ns_cluster}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02b270 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_cluster","y(3) {state}", "y(4) ns_cluster","y(5) <0.54.0>", "0x00002aaaab02b2a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,4181}, {total_heap_size,4181}, {links,[<0.54.0>]}, {memory,34208}, {message_queue_len,0}, {reductions,1600}, {trap_exit,false}]}, {<0.65.0>, [{registered_name,mb_mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796639ec0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mb_mnesia_sup},one_for_one,[{child,<0.67.0>,mb_mnesia,{mb_mnesia,st", "y(4) mb_mnesia_sup","y(5) <0.54.0>", "0x00002b8796639ef8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.66.0>,<0.67.0>,<0.54.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<0.66.0>, [{registered_name,mb_mnesia_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb3698 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) mb_mnesia_events","y(4) <0.65.0>", "0x00002aaaaadb36c8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.65.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,32}, {trap_exit,true}]}, {<0.67.0>, [{registered_name,mb_mnesia}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbfbea8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_mnesia", "y(3) {state,['ns_1@10.1.5.193']}", "y(4) mb_mnesia","y(5) <0.65.0>", "0x00002aaaabbfbee0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<0.65.0>,<0.76.0>]}, {memory,67696}, {message_queue_len,0}, {reductions,4697}, {trap_exit,true}]}, {<0.70.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaeacd40 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.71.0>,{appl_data,mnesia,[mnesia_dumper_load_regulator,mnesia_event,mnes", "y(2) <0.7.0>", "0x00002aaaaaeacd60 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.7.0>,<0.71.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<0.71.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcb310 Return addr 0x0000000000816eb8 ()", "y(0) {normal,[]}","y(1) mnesia_sup", "y(2) <0.72.0>","y(3) <0.70.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.70.0>,<0.72.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<0.72.0>, [{registered_name,mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae98ea8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_sup},one_for_all,[{child,<0.74.0>,mnesia_kernel_sup,{mnesia_", "y(4) mnesia_sup","y(5) <0.71.0>", "0x00002aaaaae98ee0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.73.0>,<0.74.0>,<0.71.0>]}, {memory,6872}, {message_queue_len,0}, {reductions,207}, {trap_exit,true}]}, {<0.73.0>, [{registered_name,mnesia_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796631680 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,mnesia_event,false,{state,[],false,[]},false}]", "y(3) mnesia_event","y(4) <0.72.0>", "0x00002b87966316b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<0.72.0>,<0.76.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,408}, {trap_exit,true}]}, {<0.74.0>, [{registered_name,mnesia_kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796632990 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_kernel_sup},one_for_all,[{child,<0.83.0>,mnesia_late_loader,", "y(4) mnesia_kernel_sup","y(5) <0.72.0>", "0x00002b87966329c8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,610}, {total_heap_size,987}, {links, [<0.77.0>,<0.81.0>,<0.82.0>,<0.83.0>,<0.79.0>, <0.80.0>,<0.78.0>,<0.75.0>,<0.76.0>,<0.72.0>]}, {memory,9016}, {message_queue_len,0}, {reductions,599}, {trap_exit,true}]}, {<0.75.0>, [{registered_name,mnesia_monitor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcfd368 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_monitor", "y(3) {state,<0.74.0>,[],[],true,[],undefined,[]}", "y(4) mnesia_monitor","y(5) <0.74.0>", "0x00002aaaabcfd3a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,26}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<0.89.0>,<0.74.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,6395}, {trap_exit,true}]}, {<0.76.0>, [{registered_name,mnesia_subscr}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb4530 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_subscr", "y(3) {state,<0.74.0>,20503}", "y(4) mnesia_subscr","y(5) <0.74.0>", "0x00002aaaaadb4568 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.73.0>,<0.74.0>,<0.67.0>]}, {memory,2704}, {message_queue_len,0}, {reductions,111}, {trap_exit,true}]}, {<0.77.0>, [{registered_name,mnesia_locker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab2ac880 (mnesia_locker:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbff068 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []", "y(5) {state,<0.74.0>}", "0x00002aaaabbff0a0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_locker","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002aaaabbff0d0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,389}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.74.0>]}, {memory,16552}, {message_queue_len,0}, {reductions,200087}, {trap_exit,true}]}, {<0.78.0>, [{registered_name,mnesia_recover}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0ff538 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_recover", "y(3) {state,<0.74.0>,undefined,undefined,undefined,0,true,[]}", "y(4) mnesia_recover","y(5) <0.74.0>", "0x00002aaaab0ff570 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,12}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<0.74.0>,<0.50.0>]}, {memory,11712}, {message_queue_len,0}, {reductions,2890}, {trap_exit,true}]}, {<0.79.0>, [{registered_name,mnesia_tm}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab2bcea0 (mnesia_tm:doit_loop/1 + 216)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796660f10 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,{0,nil},{0,nil},<0.74.0>,[],[],[]}", "y(3) []","y(4) []","y(5) <0.74.0>", "y(6) {0,nil}","y(7) {0,nil}", "0x00002b8796660f58 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_tm","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002b8796660f88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,155}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.74.0>]}, {memory,16552}, {message_queue_len,0}, {reductions,460036}, {trap_exit,true}]}, {<0.80.0>, [{registered_name,mnesia_checkpoint_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796681ea0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_checkpoint_sup},simple_one_for_one,[{child,undefined,mnesia_", "y(4) mnesia_checkpoint_sup","y(5) <0.74.0>", "0x00002b8796681ed8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<0.81.0>, [{registered_name,mnesia_snmp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879665dd60 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_snmp_sup},simple_one_for_one,[{child,undefined,mnesia_snmp_s", "y(4) mnesia_snmp_sup","y(5) <0.74.0>", "0x00002b879665dd98 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<0.82.0>, [{registered_name,mnesia_controller}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb2c80 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_controller", "y(3) {state,<0.74.0>,true,[],[],{0,nil},[],[],{0,nil},undefined,[],[],{interval,#Ref<0", "y(4) mnesia_controller","y(5) <0.74.0>", "0x00002aaaaadb2cb8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,610}, {total_heap_size,987}, {links,[<0.74.0>,<0.50.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,939}, {trap_exit,true}]}, {<0.83.0>, [{registered_name,mnesia_late_loader}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab2a8d68 (mnesia_late_loader:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966369e8 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,<0.74.0>}", "0x00002b8796636a08 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_late_loader","y(2) []", "y(3) []","y(4) [<0.74.0>]", "0x00002b8796636a38 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,377}, {total_heap_size,377}, {links,[<0.74.0>]}, {memory,3776}, {message_queue_len,0}, {reductions,172}, {trap_exit,false}]}, {<0.87.0>, [{registered_name,disk_log_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc95a8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,disk_log_sup},simple_one_for_one,[{child,undefined,disk_log,{disk_l", "y(4) disk_log_sup","y(5) <0.27.0>", "0x00002aaaaadc95e0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<0.89.0>,<0.27.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,3508}, {trap_exit,true}]}, {<0.88.0>, [{registered_name,disk_log_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02dd20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) disk_log_server","y(3) {state,[]}", "y(4) disk_log_server","y(5) <0.27.0>", "0x00002aaaab02dd58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,47}]}, {heap_size,987}, {total_heap_size,1974}, {links,[<0.89.0>,<0.27.0>]}, {memory,16672}, {message_queue_len,0}, {reductions,3851}, {trap_exit,true}]}, {<0.89.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab340f40 (disk_log:loop/1 + 168)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0d7688 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) {state,[],[],<0.87.0>,<0.88.0>,200,{arg,latest_log,undefined,\"/var/membase/mnesia", "0x00002aaaab0d7698 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,1597}, {links,[<0.87.0>,<0.88.0>,<0.75.0>,#Port<0.3053>]}, {memory,13656}, {message_queue_len,0}, {reductions,144508}, {trap_exit,true}]}, {<0.91.0>, [{registered_name,dets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0fa468 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,dets_sup},simple_one_for_one,[{child,undefined,dets,{dets,istart_li", "y(4) dets_sup","y(5) <0.27.0>", "0x00002aaaab0fa4a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.27.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,1139}, {trap_exit,true}]}, {<0.92.0>, [{registered_name,dets}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02e470 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dets_server", "y(3) {state,40996,[<0.27.0>],[]}", "y(4) dets","y(5) <0.27.0>", "0x00002aaaab02e4a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,35}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.27.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,1756}, {trap_exit,true}]}, {<0.117.0>, [{registered_name,ns_config_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae51cd8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_config_sup},rest_for_one,[{child,undefined,ns_config_log,{ns_con", "y(4) ns_config_sup","y(5) <0.54.0>", "0x00002aaaaae51d10 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<0.118.0>,<0.119.0>,<0.54.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,580}, {trap_exit,true}]}, {<0.118.0>, [{registered_name,ns_config_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x0000000000816eb0 (unknown function)", "CP: 0x0000000000816eb8 ()", "arity = 3"," proc_lib"," wake_up", " [gen_event,wake_hib,[<0.117.0>,ns_config_events,[{handler,ns_pubsub,#Ref<0.0.0.7395>,{s"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,774}, {total_heap_size,774}, {links,[<0.446.0>,<0.477.0>,<0.482.0>,<0.117.0>]}, {memory,7216}, {message_queue_len,0}, {reductions,283914}, {trap_exit,true}]}, {<0.119.0>, [{registered_name,ns_config}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac3fb868 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config", "y(3) {config,{full,\"/etc/membase/priv/config\",undefined,ns_config_default},[[{director", "y(4) ns_config","y(5) <0.117.0>", "0x00002aaaac3fb8a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,75025}, {total_heap_size,150050}, {links,[<0.117.0>]}, {memory,1201160}, {message_queue_len,0}, {reductions,96887}, {trap_exit,true}]}, {<0.422.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_setup,6}}, {backtrace, ["Program counter: 0x00002aaaabe32230 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879665b598 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,8573,4242,3,3}","y(4) normal", "y(5) 'ns_1@10.1.5.193'", "y(6) {net_address,{{10,1,5,192},21100},\"10.1.5.192\",tcp,inet}", "y(7) #Port<0.2534>", "y(8) 'ns_1@10.1.5.192'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<0.61.0>,#Port<0.2534>]}, {memory,11608}, {message_queue_len,0}, {reductions,1756}, {trap_exit,false}]}, {<0.427.0>, [{registered_name,ns_server_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb3f868 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_sup},one_for_one,[{child,<0.490.0>,moxi_stats_collector,{", "y(4) ns_server_sup","y(5) <0.54.0>", "0x00002aaaabb3f8a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,75025}, {total_heap_size,196418}, {links, [<0.462.0>,<0.476.0>,<0.481.0>,<0.482.0>,<0.490.0>, <0.479.0>,<0.480.0>,<0.477.0>,<0.474.0>,<0.475.0>, <0.468.0>,<0.430.0>,<0.459.0>,<0.460.0>,<0.446.0>, <0.428.0>,<0.429.0>,<0.54.0>]}, {memory,1572784}, {message_queue_len,0}, {reductions,87800}, {trap_exit,true}]}, {<0.428.0>, [{registered_name,ns_log}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabe05ee0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) ns_log", "y(3) {state,[{log_entry,{1302,303484,678630},'ns_1@10.1.5.193',ns_node_disco,3,\"Initia", "y(4) ns_log","y(5) <0.427.0>", "0x00002aaaabe05f18 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,4181}, {total_heap_size,10946}, {links,[<0.50.0>,<0.427.0>]}, {memory,88368}, {message_queue_len,0}, {reductions,4007}, {trap_exit,false}]}, {<0.429.0>, [{registered_name,ns_log_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0f98a0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) ns_log_events","y(4) <0.427.0>", "0x00002aaaab0f98d0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.427.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,56}, {trap_exit,true}]}, {<0.430.0>, [{registered_name,ns_node_disco_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b879662e738 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_node_disco_sup},rest_for_one,[{child,<0.439.0>,ns_config_rep,{ns", "y(4) ns_node_disco_sup","y(5) <0.427.0>", "0x00002b879662e770 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<0.432.0>,<0.439.0>,<0.431.0>,<0.427.0>]}, {memory,5760}, {message_queue_len,0}, {reductions,793}, {trap_exit,true}]}, {<0.431.0>, [{registered_name,ns_node_disco_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x0000000000816eb0 (unknown function)", "CP: 0x0000000000816eb8 ()", "arity = 3"," proc_lib"," wake_up", " [gen_event,wake_hib,[<0.430.0>,ns_node_disco_events,[{handler,menelaus_event,ns_node_di"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,73}, {total_heap_size,73}, {links,[<0.430.0>]}, {memory,1488}, {message_queue_len,0}, {reductions,128}, {trap_exit,true}]}, {<0.432.0>, [{registered_name,ns_node_disco}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabf59f68 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_node_disco", "y(3) {state,['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193'],{interval,#Ref<0.0", "y(4) ns_node_disco","y(5) <0.430.0>", "0x00002aaaabf59fa0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,45}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<0.430.0>,<0.50.0>]}, {memory,601000}, {message_queue_len,0}, {reductions,4471}, {trap_exit,false}]}, {<0.439.0>, [{registered_name,ns_config_rep}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac3308e0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config_rep","y(3) {state}", "y(4) ns_config_rep","y(5) <0.430.0>", "0x00002aaaac330918 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,46368}, {total_heap_size,64079}, {links,[<0.430.0>]}, {memory,513392}, {message_queue_len,0}, {reductions,19118}, {trap_exit,false}]}, {<0.446.0>, [{registered_name,mb_master}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab69a340 (gen_fsm:loop/7 + 312)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabe1e6e0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_master", "y(3) {state,undefined,'ns_1@10.1.5.192',['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.", "y(4) candidate","y(5) mb_master", "y(6) <0.427.0>", "0x00002aaaabe1e720 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,95}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.118.0>,<0.427.0>,<0.50.0>]}, {memory,16632}, {message_queue_len,0}, {reductions,42531}, {trap_exit,false}]}, {<0.459.0>, [{registered_name,ns_heart}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab015c50 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_heart", "y(3) [{meminfo,<<777 bytes>>},{replication,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9c", "y(4) ns_heart","y(5) <0.427.0>", "0x00002aaaab015c88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<0.427.0>,<0.50.0>]}, {memory,284176}, {message_queue_len,0}, {reductions,1396747}, {trap_exit,false}]}, {<0.460.0>, [{registered_name,ns_doctor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdf5990 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_doctor", "y(3) {state,{dict,3,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{", "y(4) ns_doctor","y(5) <0.427.0>", "0x00002aaaabdf59c8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<0.427.0>,<0.50.0>]}, {memory,142488}, {message_queue_len,0}, {reductions,57594}, {trap_exit,false}]}, {<0.462.0>, [{registered_name,menelaus_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb695f8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,menelaus_sup},one_for_one,[{child,<0.467.0>,menelaus_web_alerts_srv", "y(4) menelaus_sup","y(5) <0.427.0>", "0x00002aaaabb69630 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<0.463.0>,<0.467.0>,<0.1191.0>,<0.427.0>]}, {memory,284256}, {message_queue_len,0}, {reductions,10858}, {trap_exit,true}]}, {<0.463.0>, [{registered_name,menelaus_web}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab100f78 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mochiweb_socket_server", "y(3) {mochiweb_socket_server,8091,#Fun,{local,menelaus_web}", "y(4) menelaus_web","y(5) <0.462.0>", "0x00002aaaab100fb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,987}, {links, [<0.486.0>,<0.487.0>,<0.1192.0>,<0.462.0>,<0.464.0>, #Port<0.2542>]}, {memory,8856}, {message_queue_len,0}, {reductions,365}, {trap_exit,true}]}, {<0.464.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab6fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabddb258 Return addr 0x00002aaaab6f9b78 (menelaus_web:loop/3 + 26376)", "y(0) {struct,[{buckets,[]}]}", "y(1) {struct,[{buckets,[]}]}", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBuck", "y(3) {mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(4) #Fun", "0x00002aaaabddb288 Return addr 0x00002aaaab6d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(5) Catch 0x00002aaaab6f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaabddb2c0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2549>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "0x00002aaaabddb2e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,28657}, {total_heap_size,28657}, {links,[<0.463.0>,#Port<0.2549>]}, {memory,230200}, {message_queue_len,0}, {reductions,5372}, {trap_exit,false}]}, {<0.467.0>, [{registered_name,menelaus_web_alerts_srv}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb5c288 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) menelaus_web_alerts_srv", "y(3) {state,[],[],{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],", "y(4) menelaus_web_alerts_srv", "y(5) <0.462.0>", "0x00002aaaabb5c2c0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,324}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<0.50.0>,<0.462.0>]}, {memory,67696}, {message_queue_len,0}, {reductions,189041}, {trap_exit,false}]}, {<0.468.0>, [{registered_name,ns_port_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabd48a18 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_port_sup},one_for_one,[{child,<0.492.0>,{memcached,\"./bin/memcac", "y(4) ns_port_sup","y(5) <0.427.0>", "0x00002aaaabd48a50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<0.470.0>,<0.492.0>,<0.427.0>]}, {memory,601040}, {message_queue_len,0}, {reductions,3364}, {trap_exit,true}]}, {<0.470.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796644c20 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi,5000,{1302,303828,150702},<0.471.0>}", "y(4) <0.470.0>","y(5) <0.468.0>", "0x00002b8796644c58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<0.468.0>,<0.471.0>]}, {memory,21472}, {message_queue_len,0}, {reductions,150}, {trap_exit,true}]}, {<0.471.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab151720 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2543>,moxi,{[\"2011-04-08 16:03:48: (cproxy_config.c.325) env: MOXI", "y(4) <0.471.0>","y(5) <0.470.0>", "0x00002aaaab151758 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<0.470.0>,#Port<0.2543>]}, {memory,26352}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<0.474.0>, [{registered_name,ns_tick_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1bc10 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7346>,{state,#Fun,ignored},<", "y(3) ns_tick_event","y(4) <0.427.0>", "0x00002aaaaae1bc40 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,25}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.479.0>,<0.427.0>]}, {memory,16592}, {message_queue_len,0}, {reductions,10207}, {trap_exit,true}]}, {<0.475.0>, [{registered_name,ns_stats_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965ce9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabd0ebe0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7384>,{state,#Fun,ignored},<", "y(3) ns_stats_event","y(4) <0.427.0>", "0x00002aaaabd0ec10 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,401}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<0.480.0>,<0.427.0>]}, {memory,26352}, {message_queue_len,0}, {reductions,26035}, {trap_exit,true}]}, {<0.476.0>, [{registered_name,ns_bucket_worker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae96160 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) work_queue","y(3) []", "y(4) ns_bucket_worker","y(5) <0.427.0>", "0x00002aaaaae96198 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,1597}, {links,[<0.427.0>]}, {memory,13536}, {message_queue_len,0}, {reductions,282}, {trap_exit,false}]}, {<0.477.0>, [{registered_name,ns_bucket_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabca8e18 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_bucket_sup},one_for_one,[{child,<0.816.0>,{per_bucket_sup,\"Failo", "y(4) ns_bucket_sup","y(5) <0.427.0>", "0x00002aaaabca8e50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<0.427.0>,<0.816.0>,<0.118.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,244}, {trap_exit,true}]}, {<0.479.0>, [{registered_name,system_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0d6348 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) system_stats_collector", "y(3) {state,#Port<0.2546>,[{cpu_local_ms,1970822690},{cpu_idle_ms,1810808840},{swap_to", "y(4) system_stats_collector", "y(5) <0.427.0>", "0x00002aaaab0d6380 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,143}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.427.0>,<0.474.0>,#Port<0.2546>]}, {memory,16632}, {message_queue_len,0}, {reductions,47638}, {trap_exit,false}]}, {<0.480.0>, [{registered_name,'stats_archiver-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcfb488 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"@system\"}", "y(4) 'stats_archiver-@system'", "y(5) <0.427.0>", "0x00002aaaabcfb4c0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,880}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<0.427.0>,<0.475.0>,<0.50.0>]}, {memory,109080}, {message_queue_len,0}, {reductions,4254285}, {trap_exit,false}]}, {<0.481.0>, [{registered_name,'stats_reader-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966359b8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"@system\"}", "y(4) 'stats_reader-@system'", "y(5) <0.427.0>", "0x00002b87966359f0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.427.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,26}, {trap_exit,false}]}, {<0.482.0>, [{registered_name,ns_moxi_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabccb798 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_moxi_sup},one_for_one,[{child,<0.483.0>,{\"FailoverNodesTests-4a4", "y(4) ns_moxi_sup","y(5) <0.427.0>", "0x00002aaaabccb7d0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<0.427.0>,<0.483.0>,<0.118.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,2326}, {trap_exit,true}]}, {<0.483.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796666080 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2547>,moxi,{[\"2011-04-08 16:03:48: (cproxy_config.c.316) env: MOXI", "y(4) <0.483.0>","y(5) <0.482.0>", "0x00002b87966660b8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<0.482.0>,#Port<0.2547>]}, {memory,13576}, {message_queue_len,0}, {reductions,168}, {trap_exit,true}]}, {<0.486.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab6fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0514b8 Return addr 0x00002aaaab75fd20 (menelaus_web_buckets:checking_bucket_ac", "y(0) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(1) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsS", "y(3) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) #Fun", "0x00002aaaab0514e8 Return addr 0x00002aaaab6f9a28 (menelaus_web:loop/3 + 26040)", "y(0) #Ref<0.0.0.7487>", "y(1) Catch 0x00002aaaab75fd80 (menelaus_web_buckets:checking_bucket_access/4 + 616)", "y(2) []", "y(3) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) []", "0x00002aaaab051518 Return addr 0x00002aaaab6d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(5) Catch 0x00002aaaab6f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaab051550 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2550>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "0x00002aaaab051578 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,17711}, {total_heap_size,92736}, {links,[<0.463.0>,#Port<0.2550>]}, {memory,742832}, {message_queue_len,0}, {reductions,1282935}, {trap_exit,false}]}, {<0.487.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfd670 (gen_server:do_multi_call/4 + 768)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbbbba8 Return addr 0x00002aaaaac69fe8 (rpc:do_multicall/5 + 320)", "y(0) []","y(1) #Ref<0.0.0.24793>", "y(2) <0.1193.0>","y(3) #Ref<0.0.0.24792>", "0x00002aaaabbbbbd0 Return addr 0x00002aaaab4fe5b8 (diag_handler:diag_multicall/3 + 232)", "y(0) []","y(1) []", "0x00002aaaabbbbbe8 Return addr 0x00002aaaab4ff1d8 (diag_handler:handle_diag/1 + 384)", "y(0) ['ns_1@10.1.5.193','ns_1@10.1.5.192','ns_1@10.1.5.178']", "y(1) []","y(2) []","y(3) []", "0x00002aaaabbbbc10 Return addr 0x00002aaaab6f9620 (menelaus_web:loop/3 + 25008)", "y(0) []", "y(1) [[\"2011\",45,[\"0\",\"4\"],45,[\"0\",\"8\"],32,\"15\",58,\"58\",58,[\"0\",\"4\"],46,\"678\"],32,\"ns_", "y(2) [{\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",[{num_replicas,1},{ra", "y(3) {mochiweb_request,#Port<0.3105>,'GET',\"/diag\",{1,0},{5,{\"user-agent\",{'User-Agent", "0x00002aaaabbbbc38 Return addr 0x00002aaaab6d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.3105>,'GET',\"/diag\",{1,0},{5,{\"user-agent\",{'User-Agent", "y(5) Catch 0x00002aaaab6f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaabbbbc70 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.3105>,'GET',\"/diag\",{1,0},{5,{\"user-agent\",{'User-Agent", "0x00002aaaabbbbc98 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,17711}, {total_heap_size,46368}, {links,[<0.463.0>,#Port<0.3105>]}, {memory,371888}, {message_queue_len,0}, {reductions,23017}, {trap_exit,false}]}, {<0.490.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabe20d08 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi_stats_collector,5000,{1302,303833,493340},<0.491.0>}", "y(4) <0.490.0>","y(5) <0.427.0>", "0x00002aaaabe20d40 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.427.0>,<0.491.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<0.491.0>, [{registered_name,moxi_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0e0488 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) []","y(1) infinity", "y(2) moxi_stats_collector", "y(3) {state,#Port<0.2554>,{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[", "y(4) <0.491.0>","y(5) <0.490.0>", "0x00002aaaab0e04c0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) moxi_stats_collector","y(2) []", "y(3) moxi_stats_collector", "y(4) {local,moxi_stats_collector}", "y(5) <0.490.0>","y(6) <0.490.0>", "0x00002aaaab0e0500 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,15}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[#Port<0.2636>,<0.490.0>,#Port<0.2554>]}, {memory,263544}, {message_queue_len,0}, {reductions,2237829}, {trap_exit,false}]}, {<0.492.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb1968 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,memcached,5000,{1302,303834,858055},<0.493.0>}", "y(4) <0.492.0>","y(5) <0.468.0>", "0x00002aaaaadb19a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<0.468.0>,<0.493.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,55}, {trap_exit,true}]}, {<0.493.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab153600 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2556>,memcached,{[empty],[empty,empty]},undefined,[],0}", "y(4) <0.493.0>","y(5) <0.492.0>", "0x00002aaaab153638 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<0.492.0>,#Port<0.2556>]}, {memory,8696}, {message_queue_len,0}, {reductions,58}, {trap_exit,true}]}, {<0.500.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_setup,6}}, {backtrace, ["Program counter: 0x00002aaaabe32230 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0fd648 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,2012,1997,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.193'", "y(6) {net_address,{{10,1,5,178},21100},\"10.1.5.178\",tcp,inet}", "y(7) #Port<0.2561>", "y(8) 'ns_1@10.1.5.178'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<0.61.0>,#Port<0.2561>]}, {memory,16488}, {message_queue_len,0}, {reductions,1737}, {trap_exit,false}]}, {<0.816.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b87966007a0 (proc_lib:sync_wait/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb3df8 Return addr 0x00002aaaab78d348 (single_bucket_sup:'-start_link/1-fun-0-", "y(0) infinity","y(1) <0.817.0>", "0x00002aaaaadb3e10 Return addr 0x0000000000816eb8 ()", "y(0) <0.477.0>", "y(1) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\""]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.477.0>,<0.817.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,30}, {trap_exit,true}]}, {<0.817.0>, [{registered_name, 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87966007a0 (proc_lib:sync_wait/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae31d90 Return addr 0x00002aaaaac0ffe8 (supervisor:do_start_child/2 + 312)", "y(0) infinity","y(1) <0.818.0>", "0x00002aaaaae31da8 Return addr 0x00002aaaaac0fa28 (supervisor:start_children/3 + 152)", "y(0) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(2) Catch 0x00002aaaaac0ffe8 (supervisor:do_start_child/2 + 312)", "0x00002aaaaae31dc8 Return addr 0x00002aaaaac0f410 (supervisor:init_children/2 + 272)", "y(0) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(1) []", "y(2) [{child,undefined,{ns_vbm_sup,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b9582", "y(3) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "0x00002aaaaae31df0 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) {state,{local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "0x00002aaaaae31e08 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(2) []","y(3) supervisor", "y(4) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(5) <0.816.0>","y(6) <0.816.0>", "0x00002aaaaae31e48 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links,[<0.816.0>,<0.818.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,664}, {trap_exit,true}]}, {<0.818.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaaee3ea8 (timer:sleep/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcee0b8 Return addr 0x00002aaaab6c2278 (ns_memcached:ensure_bucket/2 + 912)", "y(0) 1000", "0x00002aaaabcee0c8 Return addr 0x00002aaaab6be190 (ns_memcached:init/1 + 144)", "y(0) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "y(1) #Port<0.2779>","y(2) []", "0x00002aaaabcee0e8 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) []","y(1) {1302,304035,74311}", "y(2) #Port<0.2779>", "y(3) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "0x00002aaaabcee110 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) <0.818.0>","y(2) []", "y(3) ns_memcached","y(4) <0.818.0>", "y(5) <0.817.0>","y(6) <0.817.0>", "0x00002aaaabcee150 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,608}]}, {heap_size,17711}, {total_heap_size,46368}, {links,[<0.817.0>,#Port<0.2779>]}, {memory,371744}, {message_queue_len,0}, {reductions,1379890}, {trap_exit,false}]}, {<0.1191.0>, [{registered_name,hot_keys_keeper}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcaba0 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) hot_keys_keeper", "y(3) {state,[],undefined,undefined}", "y(4) hot_keys_keeper","y(5) <0.462.0>", "0x00002aaaaadcabd8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.462.0>,<0.50.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,41}, {trap_exit,false}]}, {<0.1192.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b87965b7ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b87966483b0 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 2986","y(1) #Port<0.2542>", "0x00002b87966483c8 Return addr 0x00002aaaab6dfe88 (mochiweb_socket_server:acceptor_loop/1 ", "y(0) []", "0x00002b87966483d8 Return addr 0x00002b87965fff38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) <0.463.0>", "y(2) Catch 0x00002aaaab6dfe88 (mochiweb_socket_server:acceptor_loop/1 + 160)", "0x00002b87966483f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b87965fff58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<0.463.0>]}, {memory,2696}, {message_queue_len,0}, {reductions,38}, {trap_exit,false}]}, {<0.1193.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaabfdc90 (gen_server:rec_nodes/7 + 240)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b8796689540 Return addr 0x00002aaaaac03dc8 (gen_server:'-do_multi_call/4-fun-0-'/6 ", "y(0) #Ref<0.0.0.24798>","y(1) 2000", "y(2) []","y(3) []","y(4) rex", "y(5) #Ref<0.0.0.24792>", "y(6) [{'ns_1@10.1.5.192',#Ref<0.0.0.24796>},{'ns_1@10.1.5.193',#Ref<0.0.0.24795>}]", "y(7) #Ref<0.0.0.24797>", "y(8) 'ns_1@10.1.5.178'", "0x00002b8796689590 Return addr 0x0000000000816eb8 ()", "y(0) #Ref<0.0.0.24792>","y(1) []", "y(2) []","y(3) []","y(4) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[]}, {memory,2840}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<0.1194.0>, [{registered_name,[]}, {status,running}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b879653c8d8 (unknown function)", "CP: 0x00002aaaab4fe068 (diag_handler:grab_process_info/1 + 64)", "0x00002aaaaccfcd28 Return addr 0x00002aaaab500610 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []", "0x00002aaaaccfcd38 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []","y(1) <0.1194.0>", "0x00002aaaaccfcd50 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.1193.0>", "0x00002aaaaccfcd68 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.1192.0>", "0x00002aaaaccfcd80 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,hot_keys_keeper},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.1191.0>", "0x00002aaaaccfcd98 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.818.0>", "0x00002aaaaccfcdb0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-5", "y(1) <0.817.0>", "0x00002aaaaccfcdc8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.816.0>", "0x00002aaaaccfcde0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_setup,6}},", "y(1) <0.500.0>", "0x00002aaaaccfcdf8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.493.0>", "0x00002aaaaccfce10 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.492.0>", "0x00002aaaaccfce28 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,moxi_stats_collector},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.491.0>", "0x00002aaaaccfce40 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.490.0>", "0x00002aaaaccfce58 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.487.0>", "0x00002aaaaccfce70 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.486.0>", "0x00002aaaaccfce88 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.483.0>", "0x00002aaaaccfcea0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_moxi_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.482.0>", "0x00002aaaaccfceb8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_reader-@system'},{status,waiting},{initial_call,{proc_li", "y(1) <0.481.0>", "0x00002aaaaccfced0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_archiver-@system'},{status,waiting},{initial_call,{proc_", "y(1) <0.480.0>", "0x00002aaaaccfcee8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,system_stats_collector},{status,waiting},{initial_call,{proc_li", "y(1) <0.479.0>", "0x00002aaaaccfcf00 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.477.0>", "0x00002aaaaccfcf18 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_worker},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.476.0>", "0x00002aaaaccfcf30 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_stats_event},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.475.0>", "0x00002aaaaccfcf48 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_tick_event},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.474.0>", "0x00002aaaaccfcf60 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.471.0>", "0x00002aaaaccfcf78 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.470.0>", "0x00002aaaaccfcf90 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_port_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.468.0>", "0x00002aaaaccfcfa8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web_alerts_srv},{status,waiting},{initial_call,{proc_l", "y(1) <0.467.0>", "0x00002aaaaccfcfc0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.464.0>", "0x00002aaaaccfcfd8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.463.0>", "0x00002aaaaccfcff0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.462.0>", "0x00002aaaaccfd008 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_doctor},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.460.0>", "0x00002aaaaccfd020 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_heart},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.459.0>", "0x00002aaaaccfd038 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_master},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.446.0>", "0x00002aaaaccfd050 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_rep},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.439.0>", "0x00002aaaaccfd068 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.432.0>", "0x00002aaaaccfd080 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_events},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.431.0>", "0x00002aaaaccfd098 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.430.0>", "0x00002aaaaccfd0b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log_events},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.429.0>", "0x00002aaaaccfd0c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.428.0>", "0x00002aaaaccfd0e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.427.0>", "0x00002aaaaccfd0f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_setup,6}},", "y(1) <0.422.0>", "0x00002aaaaccfd110 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.119.0>", "0x00002aaaaccfd128 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.118.0>", "0x00002aaaaccfd140 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.117.0>", "0x00002aaaaccfd158 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.92.0>", "0x00002aaaaccfd170 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.91.0>", "0x00002aaaaccfd188 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.89.0>", "0x00002aaaaccfd1a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_server},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.88.0>", "0x00002aaaaccfd1b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.87.0>", "0x00002aaaaccfd1d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_late_loader},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.83.0>", "0x00002aaaaccfd1e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_controller},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.82.0>", "0x00002aaaaccfd200 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_snmp_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.81.0>", "0x00002aaaaccfd218 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_checkpoint_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.80.0>", "0x00002aaaaccfd230 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_tm},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.79.0>", "0x00002aaaaccfd248 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_recover},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.78.0>", "0x00002aaaaccfd260 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_locker},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.77.0>", "0x00002aaaaccfd278 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_subscr},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.76.0>", "0x00002aaaaccfd290 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_monitor},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.75.0>", "0x00002aaaaccfd2a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_kernel_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.74.0>", "0x00002aaaaccfd2c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_event},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.73.0>", "0x00002aaaaccfd2d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.72.0>", "0x00002aaaaccfd2f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.71.0>", "0x00002aaaaccfd308 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.70.0>", "0x00002aaaaccfd320 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.67.0>", "0x00002aaaaccfd338 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.66.0>", "0x00002aaaaccfd350 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.65.0>", "0x00002aaaaccfd368 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_cluster},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.64.0>", "0x00002aaaaccfd380 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{net_kernel,ticker,2}},{back", "y(1) <0.63.0>", "0x00002aaaaccfd398 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,accept_loop,2", "y(1) <0.62.0>", "0x00002aaaaccfd3b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_kernel},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.61.0>", "0x00002aaaaccfd3c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,auth},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.60.0>", "0x00002aaaaccfd3e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_epmd},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.59.0>", "0x00002aaaaccfd3f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.58.0>", "0x00002aaaaccfd410 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native},{status,waiting},{initial_call,{inet_getho", "y(1) <0.57.0>", "0x00002aaaaccfd428 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native_sup},{status,waiting},{initial_call,{proc_l", "y(1) <0.56.0>", "0x00002aaaaccfd440 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dist_manager},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.55.0>", "0x00002aaaaccfd458 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_cluster_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.54.0>", "0x00002aaaaccfd470 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.53.0>", "0x00002aaaaccfd488 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.52.0>", "0x00002aaaaccfd4a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,timer_server},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.50.0>", "0x00002aaaaccfd4b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.49.0>", "0x00002aaaaccfd4d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.48.0>", "0x00002aaaaccfd4e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,cpu_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.47.0>", "0x00002aaaaccfd500 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.46.0>", "0x00002aaaaccfd518 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,memsup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.45.0>", "0x00002aaaaccfd530 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disksup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.44.0>", "0x00002aaaaccfd548 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,os_mon_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.43.0>", "0x00002aaaaccfd560 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.42.0>", "0x00002aaaaccfd578 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.41.0>", "0x00002aaaaccfd590 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,release_handler},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.38.0>", "0x00002aaaaccfd5a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,overload},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.37.0>", "0x00002aaaaccfd5c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,alarm_handler},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.36.0>", "0x00002aaaaccfd5d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_safe_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.35.0>", "0x00002aaaaccfd5f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.34.0>", "0x00002aaaaccfd608 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.33.0>", "0x00002aaaaccfd620 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.32.0>", "0x00002aaaaccfd638 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_safe_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.27.0>", "0x00002aaaaccfd650 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.26.0>", "0x00002aaaaccfd668 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,user},{status,waiting},{initial_call,{user,server,2}},{backtrac", "y(1) <0.25.0>", "0x00002aaaaccfd680 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.24.0>", "0x00002aaaaccfd698 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error},{status,waiting},{initial_call,{standard_error,", "y(1) <0.23.0>", "0x00002aaaaccfd6b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error_sup},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.22.0>", "0x00002aaaaccfd6c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,code_server},{status,waiting},{initial_call,{erlang,apply,2}},{", "y(1) <0.21.0>", "0x00002aaaaccfd6e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,file_server_2},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.20.0>", "0x00002aaaaccfd6f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_group},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.19.0>", "0x00002aaaaccfd710 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_db},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.17.0>", "0x00002aaaaccfd728 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.16.0>", "0x00002aaaaccfd740 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.15.0>", "0x00002aaaaccfd758 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.14.0>", "0x00002aaaaccfd770 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_name_server},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.13.0>", "0x00002aaaaccfd788 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,rex},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backt", "y(1) <0.12.0>", "0x00002aaaaccfd7a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.11.0>", "0x00002aaaaccfd7b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.10.0>", "0x00002aaaaccfd7d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.9.0>", "0x00002aaaaccfd7e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,application_controller},{status,waiting},{initial_call,{erlang,", "y(1) <0.7.0>", "0x00002aaaaccfd800 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,error_logger},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.6.0>", "0x00002aaaaccfd818 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_prim_loader},{status,waiting},{initial_call,{erlang,apply,2", "y(1) <0.3.0>", "0x00002aaaaccfd830 Return addr 0x00002aaaab4fe278 (diag_handler:do_diag_per_node/0 + 224)", "y(0) [{registered_name,init},{status,waiting},{initial_call,{otp_ring0,start,2}},{back", "y(1) <0.0.0>", "0x00002aaaaccfd848 Return addr 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 20", "y(0) []","y(1) []", "y(2) [{version,[{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"}", "y(3) [{alerts,[{email,[]},{email_alerts,false},{email_server,[{user,undefined},{pass,'", "y(4) [\"bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz\",\"curl-7.21.1-w64_patched.t", "y(5) [{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"},{ns_serve", "0x00002aaaaccfd880 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 208)", "y(1) []","y(2) []","y(3) []", "y(4) <0.12.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,75025}, {total_heap_size,589254}, {links,[]}, {memory,4714720}, {message_queue_len,0}, {reductions,687175}, {trap_exit,false}]}]}, {memory,{2058928128,1069023232,{<0.427.0>,1572784}}}, {disk, [{"/",13140720,39}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}, {'ns_1@10.1.5.192', [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {manifest, ["bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz", "curl-7.21.1-w64_patched.tar.gz", "ep-engine_1.6.5.3_143_g366c22e-Linux.x86_64.tar.gz", "google-perftools-1.7","google-perftools-1.7.tar.gz", "libconflate_1.6.5_1_g8a42194-Linux.x86_64.tar.gz", "libevent-2.0.7-rc.tar.gz", "libmemcached-0.41_trond-norbye_mingw32-revno895.tar.gz", "libvbucket_1.6.4_1_g5cb588e-Linux.x86_64.tar.gz", "membase-cli_1.6.5_2_gb9c8a7d-Linux.x86_64.tar.gz", "memcached_1.4.4_430_g6235145-Linux.x86_64.tar.gz", "moxi_1.6.5.1_16_g8128746-Linux.x86_64.tar.gz", "ns_server_1.6.5.3_291_gba5fe27.tar.gz", "portsigar_-Linux.x86_64.tar.gz", "sigar_sigar_1.6.4_402_g7809620-Linux.x86_64.tar.gz", "vbucketmigrator_1.7.0r_4_g8428c29-Linux.x86_64.tar.gz", "wallace_1.7.0r-100-g85ee8f5-Linux.x86_64-201104060842"]}, {config, [{alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,'filtered-out'}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up, server_joined,server_left,bucket_created, bucket_deleted,bucket_auth_failed]}]}, {buckets, [{'_vclock',[{'ns_1@10.1.5.192',{11,63469523235}}]}, {configs, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers, ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178']}, {map, [['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}]}]}, {memory_quota,1570}, {nodes_wanted, [{'_vclock',[{'ns_1@10.1.5.192',{2,63469523039}}]}, 'ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193']}, {otp, [{'_vclock',[{'ns_1@10.1.5.193',{1,63469523027}}]}, {cookie,lulxzmudqfmlvwvs}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {rebalance_status, {none, <<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>}}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds, [{'_vclock',[{'ns_1@10.1.5.192',{1,63469522657}}]}, {creds, [{"Administrator",[{password,'filtered-out'}]}]}]}, {{node,'ns_1@10.1.5.178',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.178',membership},inactiveFailed}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.178',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.192',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.192',membership},active}, {{node,'ns_1@10.1.5.192',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.192',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.192',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.192',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.193',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.193',membership},inactiveFailed}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.193',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}]}, {basic_info, [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1192}, {memory_data, {2058928128,1594331136,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}, {processes, [{<10461.0.0>, [{registered_name,init}, {status,waiting}, {initial_call,{otp_ring0,start,2}}, {backtrace, ["Program counter: 0x00002b2847ee2290 (init:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdb8000 Return addr 0x0000000000816eb8 ()", "y(0) {state,[{'-root',[<<37 bytes>>]},{'-progname',[<<3 bytes>>]},{'-home',[<<12 bytes"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,290}]}, {heap_size,4181}, {total_heap_size,10946}, {links,[<10461.6.0>,<10461.7.0>,<10461.3.0>]}, {memory,88304}, {message_queue_len,0}, {reductions,131540}, {trap_exit,true}]}, {<10461.3.0>, [{registered_name,erl_prim_loader}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b2847f41d20 (erl_prim_loader:loop/3 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb896f0 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) [\"/opt/membase/1.7.0r/bin/ns_server/ebin\",\"/opt/membase/1.7.0r/bin/ns_server/deps", "y(2) <0.2.0>", "y(3) {state,efile,[],none,#Port<0.2>,infinity,undefined,true,{prim_state,false,undefin", "y(4) infinity"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,331}]}, {heap_size,6765}, {total_heap_size,53133}, {links,[#Port<10461.2>,<10461.0.0>]}, {memory,425760}, {message_queue_len,0}, {reductions,1306313}, {trap_exit,true}]}, {<10461.6.0>, [{registered_name,error_logger}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae16b48 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_log_mf_h,false,{state,\"/var/membase/logs\",10485760,10,1429081,1,{fil", "y(3) error_logger","y(4) <0.2.0>", "0x00002aaaaae16b78 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,28}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10461.0.0>,<10461.25.0>,#Port<10461.981>]}, {memory,54960}, {message_queue_len,0}, {reductions,461470}, {trap_exit,true}]}, {<10461.7.0>, [{registered_name,application_controller}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcc8ae0 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) infinity", "y(2) application_controller", "y(3) {state,[],[],[],[{inets,<0.584.0>},{ns_server,<0.52.0>},{mnesia,<0.70.0>},{os_mon", "y(4) application_controller", "y(5) <0.2.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [<10461.41.0>,<10461.70.0>,<10461.584.0>,<10461.52.0>, <10461.9.0>,<10461.32.0>,<10461.0.0>]}, {memory,263704}, {message_queue_len,0}, {reductions,40425}, {trap_exit,true}]}, {<10461.9.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa62f0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.10.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_ser", "y(2) <0.7.0>", "0x00002b2847fa6310 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10461.7.0>,<10461.10.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<10461.10.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f96790 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) kernel","y(2) <0.11.0>", "y(3) <0.9.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.9.0>,<10461.11.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,72}, {trap_exit,true}]}, {<10461.11.0>, [{registered_name,kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadba358 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_sup},one_for_all,[{child,<0.58.0>,net_sup_dynamic,{erl_distr", "y(4) kernel_sup","y(5) <0.10.0>", "0x00002aaaaadba390 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10461.22.0>,<10461.26.0>,<10461.27.0>,<10461.58.0>, <10461.24.0>,<10461.17.0>,<10461.20.0>,<10461.21.0>, <10461.19.0>,<10461.12.0>,<10461.13.0>,<10461.10.0>]}, {memory,12112}, {message_queue_len,0}, {reductions,1492}, {trap_exit,true}]}, {<10461.12.0>, [{registered_name,rex}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaf343b78 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) rpc", "y(3) {1,{<0.6292.0>,{<11857.1193.0>,{#Ref<11857.0.0.24792>,'ns_1@10.1.5.192'}},nil,nil", "y(4) rex","y(5) <0.11.0>", "0x00002aaaaf343bb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,317811}, {total_heap_size,317811}, {links,[<10461.11.0>]}, {memory,2543440}, {message_queue_len,0}, {reductions,2526}, {trap_exit,true}]}, {<10461.13.0>, [{registered_name,global_name_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab03a4d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) global", "y(3) {state,true,['ns_1@10.1.5.193','ns_1@10.1.5.178'],['ns_1@10.1.5.178','ns_1@10.1.5", "y(4) global_name_server","y(5) <0.11.0>", "0x00002aaaab03a508 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,56}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10461.14.0>,<10461.16.0>,<10461.15.0>,<10461.11.0>]}, {memory,11936}, {message_queue_len,0}, {reductions,7610}, {trap_exit,true}]}, {<10461.14.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac82618 (global:loop_the_locker/1 + 1176)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae27c98 Return addr 0x00002aaaaac82140 (global:init_the_locker/1 + 384)", "y(0) {multi,[],[],['ns_1@10.1.5.178','ns_1@10.1.5.193'],'ns_1@10.1.5.193',false,false}", "y(1) infinity", "0x00002aaaaae27cb0 Return addr 0x0000000000816eb8 ()"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10461.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,759}, {trap_exit,true}]}, {<10461.15.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a770 (global:collect_deletions/2 + 152)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa8930 Return addr 0x00002aaaaac8a670 (global:loop_the_deleter/1 + 72)", "y(0) infinity","y(1) []", "y(2) <0.13.0>", "0x00002b2847fa8950 Return addr 0x0000000000816eb8 ()", "y(0) <0.13.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.13.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,4}, {trap_exit,false}]}, {<10461.16.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaac8a9b0 (global:loop_the_registrar/0 + 24)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa1808 Return addr 0x0000000000816eb8 ()", "y(0) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10461.13.0>]}, {memory,8552}, {message_queue_len,0}, {reductions,248}, {trap_exit,false}]}, {<10461.17.0>, [{registered_name,inet_db}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fd2e20 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) inet_db", "y(3) {state,inet_db,inet_cache,inet_hosts_byname,inet_hosts_byaddr,inet_hosts_file_byn", "y(4) inet_db","y(5) <0.11.0>", "0x00002b2847fd2e58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10461.11.0>]}, {memory,26312}, {message_queue_len,0}, {reductions,1457}, {trap_exit,true}]}, {<10461.19.0>, [{registered_name,global_group}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa75f8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) global_group", "y(3) {state,no_conf,true,[],[],[],[],[],'nonode@nohost',[],normal,normal}", "y(4) global_group","y(5) <0.11.0>", "0x00002b2847fa7630 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,84}, {trap_exit,true}]}, {<10461.20.0>, [{registered_name,file_server_2}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae3ece0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) file_server","y(3) #Port<0.223>", "y(4) file_server_2","y(5) <0.11.0>", "0x00002aaaaae3ed18 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,623}]}, {heap_size,4181}, {total_heap_size,15127}, {links,[#Port<10461.223>,<10461.11.0>]}, {memory,121816}, {message_queue_len,0}, {reductions,832316}, {trap_exit,true}]}, {<10461.21.0>, [{registered_name,code_server}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaad54b40 (code_server:loop/1 + 128)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2848007f08 Return addr 0x0000000000816eb8 ()", "y(0) {state,<0.11.0>,\"/opt/membase/1.7.0r/erlang/lib/erlang\",[\"/opt/membase/1.7.0r/bin", "y(1) <0.11.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,212}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10461.11.0>]}, {memory,284032}, {message_queue_len,0}, {reductions,263452}, {trap_exit,true}]}, {<10461.22.0>, [{registered_name,standard_error_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fc93d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,standard_error,<0.23.0>,<0.23.0>,{local,standard_error_sup}}", "y(4) standard_error_sup","y(5) <0.11.0>", "0x00002b2847fc9408 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.11.0>,<10461.23.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10461.23.0>, [{registered_name,standard_error}, {status,waiting}, {initial_call,{standard_error,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad203a0 (standard_error:server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fca2a8 Return addr 0x0000000000816eb8 ()", "y(0) #Port<0.506>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.22.0>,#Port<10461.506>]}, {memory,2664}, {message_queue_len,0}, {reductions,7}, {trap_exit,true}]}, {<10461.24.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb8478 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,user_sup,<0.25.0>,<0.25.0>,{<0.24.0>,user_sup}}", "y(4) <0.24.0>","y(5) <0.11.0>", "0x00002aaaaadb84b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<10461.11.0>,<10461.25.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,71}, {trap_exit,true}]}, {<10461.25.0>, [{registered_name,user}, {status,waiting}, {initial_call,{user,server,2}}, {backtrace, ["Program counter: 0x00002aaaaad765e0 (user:server_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae51130 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) {[],[]}", "y(3) #Port<0.523>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<10461.6.0>,<10461.24.0>,#Port<10461.523>]}, {memory,109080}, {message_queue_len,0}, {reductions,27428}, {trap_exit,true}]}, {<10461.26.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fc9b20 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) kernel_config","y(3) []", "y(4) <0.26.0>","y(5) <0.11.0>", "0x00002b2847fc9b58 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.11.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,176}, {trap_exit,true}]}, {<10461.27.0>, [{registered_name,kernel_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb3f88 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_safe_sup},one_for_one,[{child,<0.92.0>,dets,{dets_server,sta", "y(4) kernel_safe_sup","y(5) <0.11.0>", "0x00002aaaaadb3fc0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10461.87.0>,<10461.91.0>,<10461.92.0>,<10461.88.0>, <10461.50.0>,<10461.56.0>,<10461.11.0>]}, {memory,7032}, {message_queue_len,0}, {reductions,439}, {trap_exit,true}]}, {<10461.32.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fab240 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.33.0>,{appl_data,sasl,[sasl_sup,alarm_handler,overload,release_handler]", "y(2) <0.7.0>", "0x00002b2847fab260 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.7.0>,<10461.33.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,27}, {trap_exit,true}]}, {<10461.33.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa3df0 Return addr 0x0000000000816eb8 ()", "y(0) {state,tty,undefined}","y(1) sasl", "y(2) <0.34.0>","y(3) <0.32.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.32.0>,<10461.34.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,70}, {trap_exit,true}]}, {<10461.34.0>, [{registered_name,sasl_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff5ef8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_sup},one_for_one,[{child,<0.38.0>,release_handler,{release_han", "y(4) sasl_sup","y(5) <0.33.0>", "0x00002b2847ff5f30 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.35.0>,<10461.38.0>,<10461.33.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,276}, {trap_exit,true}]}, {<10461.35.0>, [{registered_name,sasl_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f90f50 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_safe_sup},one_for_one,[{child,<0.37.0>,overload,{overload,star", "y(4) sasl_safe_sup","y(5) <0.34.0>", "0x00002b2847f90f88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.36.0>,<10461.37.0>,<10461.34.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,174}, {trap_exit,true}]}, {<10461.36.0>, [{registered_name,alarm_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fca9c8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,alarm_handler,false,[],false}]", "y(3) alarm_handler","y(4) <0.35.0>", "0x00002b2847fca9f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,28}, {trap_exit,true}]}, {<10461.37.0>, [{registered_name,overload}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fcb860 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) overload", "y(3) {state,0,0,8.000000e-01,223,1.000000e-01,{0,0},clear}", "y(4) overload","y(5) <0.35.0>", "0x00002b2847fcb898 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.35.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,39}, {trap_exit,false}]}, {<10461.38.0>, [{registered_name,release_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea2440 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) release_handler", "y(3) {state,[],\"/opt/membase/1.7.0r/erlang/lib/erlang\",\"/opt/membase/1.7.0r/erlang/lib", "y(4) release_handler","y(5) <0.34.0>", "0x00002aaaaaea2478 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,4181}, {links,[<10461.34.0>]}, {memory,34208}, {message_queue_len,0}, {reductions,2237}, {trap_exit,false}]}, {<10461.41.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fcb128 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.42.0>,{appl_data,os_mon,[os_mon_sup,os_mon_sysinfo,disksup,memsup,cpu_s", "y(2) <0.7.0>", "0x00002b2847fcb148 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.7.0>,<10461.42.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<10461.42.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa2390 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) os_mon","y(2) <0.43.0>", "y(3) <0.41.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.41.0>,<10461.43.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10461.43.0>, [{registered_name,os_mon_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847faa808 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,os_mon_sup},one_for_one,[{child,<0.47.0>,cpu_sup,{cpu_sup,start_lin", "y(4) os_mon_sup","y(5) <0.42.0>", "0x00002b2847faa840 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10461.44.0>,<10461.45.0>,<10461.47.0>,<10461.42.0>]}, {memory,6912}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10461.44.0>, [{registered_name,disksup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab02e468 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) disksup", "y(3) {state,80,60000,{unix,linux},[{\"/\",13140720,42},{\"/boot\",101086,13},{\"/dev/shm\",1", "y(4) disksup","y(5) <0.43.0>", "0x00002aaaab02e4a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,33}]}, {heap_size,2584}, {total_heap_size,5168}, {links,[<10461.43.0>,#Port<10461.879>]}, {memory,42144}, {message_queue_len,0}, {reductions,28553}, {trap_exit,true}]}, {<10461.45.0>, [{registered_name,memsup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1120b0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) memsup", "y(3) {state,{unix,linux},true,{1594331136,2058928128},{<0.462.0>,2172304},false,60000,", "y(4) memsup","y(5) <0.43.0>", "0x00002aaaab1120e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,45}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<10461.43.0>,<10461.46.0>]}, {memory,13576}, {message_queue_len,0}, {reductions,47110}, {trap_exit,true}]}, {<10461.46.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf49870 (memsup:port_idle/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc005f0 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.908>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,37}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10461.45.0>,#Port<10461.908>]}, {memory,16488}, {message_queue_len,0}, {reductions,13188}, {trap_exit,true}]}, {<10461.47.0>, [{registered_name,cpu_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc7e18 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) cpu_sup", "y(3) {state,<0.48.0>,{unix,linux}}", "y(4) cpu_sup","y(5) <0.43.0>", "0x00002aaaaadc7e50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.43.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,33}, {trap_exit,true}]}, {<10461.48.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3a528 (cpu_sup:measurement_server_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa2ab8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []","y(5) []", "y(6) []","y(7) []", "y(8) {internal,<0.49.0>,[],{unix,linux}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.49.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,11}, {trap_exit,true}]}, {<10461.49.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaaaf3b700 (cpu_sup:port_server_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae296e8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) 6000", "y(2) #Port<0.946>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10461.48.0>,#Port<10461.946>]}, {memory,8592}, {message_queue_len,0}, {reductions,299}, {trap_exit,false}]}, {<10461.50.0>, [{registered_name,timer_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcaaab0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) 338","y(2) timer", "y(3) []","y(4) timer_server", "y(5) <0.27.0>", "0x00002aaaabcaaae8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2030}]}, {heap_size,2584}, {total_heap_size,2961}, {links, [<10461.144.0>,<10461.154.0>,<10461.448.0>, <10461.461.0>,<10461.166.0>,<10461.147.0>, <10461.153.0>,<10461.145.0>,<10461.125.0>, <10461.141.0>,<10461.143.0>,<10461.129.0>, <10461.78.0>,<10461.82.0>,<10461.27.0>]}, {memory,25008}, {message_queue_len,0}, {reductions,1389454}, {trap_exit,true}]}, {<10461.52.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9ed30 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.53.0>,{appl_data,ns_server,[ns_server_sup,ns_config,ns_config_sup,ns_co", "y(2) <0.7.0>", "0x00002b2847f9ed50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.7.0>,<10461.53.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,46}, {trap_exit,true}]}, {<10461.53.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9e5e8 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) ns_server", "y(2) <0.54.0>","y(3) <0.52.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.52.0>,<10461.54.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,50}, {trap_exit,true}]}, {<10461.54.0>, [{registered_name,ns_server_cluster_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fe5258 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_cluster_sup},one_for_one,[{child,<0.124.0>,ns_server_sup,", "y(4) ns_server_cluster_sup","y(5) <0.53.0>", "0x00002b2847fe5290 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,2584}, {total_heap_size,5168}, {links, [<10461.65.0>,<10461.117.0>,<10461.124.0>, <10461.55.0>,<10461.64.0>,<10461.53.0>]}, {memory,42304}, {message_queue_len,0}, {reductions,1840}, {trap_exit,true}]}, {<10461.55.0>, [{registered_name,dist_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa4520 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dist_manager", "y(3) {state,true,\"10.1.5.192\"}", "y(4) dist_manager","y(5) <0.54.0>", "0x00002b2847fa4558 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.54.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,912}, {trap_exit,false}]}, {<10461.56.0>, [{registered_name,inet_gethost_native_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaeac158 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,inet_gethost_native,<0.57.0>,<0.57.0>,{local,inet_gethost_native_sup}}", "y(4) inet_gethost_native_sup", "y(5) <0.27.0>", "0x00002aaaaaeac190 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.27.0>,<10461.57.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,41}, {trap_exit,true}]}, {<10461.57.0>, [{registered_name,inet_gethost_native}, {status,waiting}, {initial_call,{inet_gethost_native,server_init,2}}, {backtrace, ["Program counter: 0x00002aaaaacc9bd8 (inet_gethost_native:main_loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae40030 Return addr 0x0000000000816eb8 ()", "y(0) {state,#Port<0.1012>,8000,8209,12306,<0.56.0>,4,{statistics,0,0,0,0,0,0,0,0}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,624}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10461.56.0>,#Port<10461.1012>]}, {memory,8696}, {message_queue_len,0}, {reductions,52858}, {trap_exit,true}]}, {<10461.58.0>, [{registered_name,net_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fe7d10 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,net_sup},one_for_all,[{child,<0.61.0>,net_kernel,{net_kernel,start_", "y(4) net_sup","y(5) <0.11.0>", "0x00002b2847fe7d48 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links, [<10461.59.0>,<10461.60.0>,<10461.61.0>,<10461.11.0>]}, {memory,8776}, {message_queue_len,0}, {reductions,252}, {trap_exit,true}]}, {<10461.59.0>, [{registered_name,erl_epmd}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcb020 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) erl_epmd", "y(3) {state,#Port<0.1048>,21100,ns_1}", "y(4) erl_epmd","y(5) <0.58.0>", "0x00002aaaaadcb058 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.58.0>,#Port<10461.1048>]}, {memory,2664}, {message_queue_len,0}, {reductions,127}, {trap_exit,false}]}, {<10461.60.0>, [{registered_name,auth}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f90800 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) auth", "y(3) {state,lulxzmudqfmlvwvs,16403}", "y(4) auth","y(5) <0.58.0>", "0x00002b2847f90838 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.58.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,223}, {trap_exit,true}]}, {<10461.61.0>, [{registered_name,net_kernel}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1cc78 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) net_kernel", "y(3) {state,'ns_1@10.1.5.192','ns_1@10.1.5.192',longnames,{tick,<0.63.0>,15000},7000,s", "y(4) net_kernel","y(5) <0.58.0>", "0x00002aaaaae1ccb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,12}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10461.63.0>,<10461.604.0>,<10461.643.0>, <10461.58.0>,<10461.62.0>,#Port<10461.1046>]}, {memory,8888}, {message_queue_len,0}, {reductions,2225}, {trap_exit,true}]}, {<10461.62.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,accept_loop,2}}, {backtrace, ["Program counter: 0x00002b2847f14ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadccd60 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 8538","y(1) #Port<0.1046>", "0x00002aaaaadccd78 Return addr 0x00002aaaaaf84540 (inet_tcp_dist:accept_loop/2 + 96)", "y(0) []", "0x00002aaaaadccd88 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) #Port<0.1046>", "y(2) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.61.0>]}, {memory,2592}, {message_queue_len,0}, {reductions,268}, {trap_exit,false}]}, {<10461.63.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{net_kernel,ticker,2}}, {backtrace, ["Program counter: 0x00002aaaaad358c8 (net_kernel:ticker_loop/2 + 56)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcbef0 Return addr 0x0000000000816eb8 ()", "y(0) 15000","y(1) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.61.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,161}, {trap_exit,false}]}, {<10461.64.0>, [{registered_name,ns_cluster}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc7adb8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_cluster","y(3) {state}", "y(4) ns_cluster","y(5) <0.54.0>", "0x00002aaaabc7adf0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,17711}, {total_heap_size,46368}, {links,[<10461.54.0>]}, {memory,371704}, {message_queue_len,0}, {reductions,21119}, {trap_exit,false}]}, {<10461.65.0>, [{registered_name,mb_mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9de78 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mb_mnesia_sup},one_for_one,[{child,<0.67.0>,mb_mnesia,{mb_mnesia,st", "y(4) mb_mnesia_sup","y(5) <0.54.0>", "0x00002b2847f9deb0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.66.0>,<10461.67.0>,<10461.54.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<10461.66.0>, [{registered_name,mb_mnesia_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcd4b8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) mb_mnesia_events","y(4) <0.65.0>", "0x00002aaaaadcd4e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.65.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,32}, {trap_exit,true}]}, {<10461.67.0>, [{registered_name,mb_mnesia}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0ea650 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_mnesia", "y(3) {state,['ns_1@10.1.5.192']}", "y(4) mb_mnesia","y(5) <0.65.0>", "0x00002aaaab0ea688 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10461.65.0>,<10461.76.0>]}, {memory,54920}, {message_queue_len,0}, {reductions,4622}, {trap_exit,true}]}, {<10461.70.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9bfd8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.71.0>,{appl_data,mnesia,[mnesia_dumper_load_regulator,mnesia_event,mnes", "y(2) <0.7.0>", "0x00002b2847f9bff8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.7.0>,<10461.71.0>]}, {memory,5680}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<10461.71.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadce370 Return addr 0x0000000000816eb8 ()", "y(0) {normal,[]}","y(1) mnesia_sup", "y(2) <0.72.0>","y(3) <0.70.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.70.0>,<10461.72.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10461.72.0>, [{registered_name,mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1dad8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_sup},one_for_all,[{child,<0.74.0>,mnesia_kernel_sup,{mnesia_", "y(4) mnesia_sup","y(5) <0.71.0>", "0x00002aaaaae1db10 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10461.73.0>,<10461.74.0>,<10461.71.0>]}, {memory,6872}, {message_queue_len,0}, {reductions,207}, {trap_exit,true}]}, {<10461.73.0>, [{registered_name,mnesia_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9b878 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,mnesia_event,false,{state,[],false,[]},false}]", "y(3) mnesia_event","y(4) <0.72.0>", "0x00002b2847f9b8a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10461.72.0>,<10461.76.0>]}, {memory,6832}, {message_queue_len,0}, {reductions,408}, {trap_exit,true}]}, {<10461.74.0>, [{registered_name,mnesia_kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9ac98 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_kernel_sup},one_for_all,[{child,<0.83.0>,mnesia_late_loader,", "y(4) mnesia_kernel_sup","y(5) <0.72.0>", "0x00002b2847f9acd0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10461.77.0>,<10461.81.0>,<10461.82.0>,<10461.83.0>, <10461.79.0>,<10461.80.0>,<10461.78.0>,<10461.75.0>, <10461.76.0>,<10461.72.0>]}, {memory,9016}, {message_queue_len,0}, {reductions,599}, {trap_exit,true}]}, {<10461.75.0>, [{registered_name,mnesia_monitor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae53bc0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_monitor", "y(3) {state,<0.74.0>,[],[],true,[],undefined,[]}", "y(4) mnesia_monitor","y(5) <0.74.0>", "0x00002aaaaae53bf8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,37}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10461.89.0>,<10461.74.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,7361}, {trap_exit,true}]}, {<10461.76.0>, [{registered_name,mnesia_subscr}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcf670 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_subscr", "y(3) {state,<0.74.0>,20503}", "y(4) mnesia_subscr","y(5) <0.74.0>", "0x00002aaaaadcf6a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.73.0>,<10461.74.0>,<10461.67.0>]}, {memory,2704}, {message_queue_len,0}, {reductions,111}, {trap_exit,true}]}, {<10461.77.0>, [{registered_name,mnesia_locker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab3ac880 (mnesia_locker:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae11a48 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []", "y(5) {state,<0.74.0>}", "0x00002aaaaae11a80 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_locker","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002aaaaae11ab0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,592}]}, {heap_size,2584}, {total_heap_size,2961}, {links,[<10461.74.0>]}, {memory,24448}, {message_queue_len,0}, {reductions,318369}, {trap_exit,true}]}, {<10461.78.0>, [{registered_name,mnesia_recover}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdc5f18 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_recover", "y(3) {state,<0.74.0>,undefined,undefined,undefined,0,true,[]}", "y(4) mnesia_recover","y(5) <0.74.0>", "0x00002aaaabdc5f50 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10461.74.0>,<10461.50.0>]}, {memory,11712}, {message_queue_len,0}, {reductions,2930}, {trap_exit,true}]}, {<10461.79.0>, [{registered_name,mnesia_tm}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab3bcea0 (mnesia_tm:doit_loop/1 + 216)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadbe820 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,{0,nil},{0,nil},<0.74.0>,[],[],[]}", "y(3) []","y(4) []","y(5) <0.74.0>", "y(6) {0,nil}","y(7) {0,nil}", "0x00002aaaaadbe868 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_tm","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0x00002aaaaadbe898 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,246}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10461.74.0>]}, {memory,16552}, {message_queue_len,0}, {reductions,740143}, {trap_exit,true}]}, {<10461.80.0>, [{registered_name,mnesia_checkpoint_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f981e0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_checkpoint_sup},simple_one_for_one,[{child,undefined,mnesia_", "y(4) mnesia_checkpoint_sup","y(5) <0.74.0>", "0x00002b2847f98218 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10461.81.0>, [{registered_name,mnesia_snmp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa3210 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_snmp_sup},simple_one_for_one,[{child,undefined,mnesia_snmp_s", "y(4) mnesia_snmp_sup","y(5) <0.74.0>", "0x00002b2847fa3248 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.74.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10461.82.0>, [{registered_name,mnesia_controller}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadca8d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mnesia_controller", "y(3) {state,<0.74.0>,true,[],[],{0,nil},[],[],{0,nil},undefined,[],[],{interval,#Ref<0", "y(4) mnesia_controller","y(5) <0.74.0>", "0x00002aaaaadca908 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10461.74.0>,<10461.50.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,937}, {trap_exit,true}]}, {<10461.83.0>, [{registered_name,mnesia_late_loader}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab3a8d68 (mnesia_late_loader:loop/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f98d98 Return addr 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(0) []","y(1) []", "y(2) {state,<0.74.0>}", "0x00002b2847f98db8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaab22fa58 (mnesia_sp:init_proc/4 + 264)", "y(1) mnesia_late_loader","y(2) []", "y(3) []","y(4) [<0.74.0>]", "0x00002b2847f98de8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,377}, {total_heap_size,377}, {links,[<10461.74.0>]}, {memory,3776}, {message_queue_len,0}, {reductions,172}, {trap_exit,false}]}, {<10461.87.0>, [{registered_name,disk_log_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab09d090 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,disk_log_sup},simple_one_for_one,[{child,undefined,disk_log,{disk_l", "y(4) disk_log_sup","y(5) <0.27.0>", "0x00002aaaab09d0c8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<10461.89.0>,<10461.27.0>]}, {memory,26352}, {message_queue_len,0}, {reductions,4499}, {trap_exit,true}]}, {<10461.88.0>, [{registered_name,disk_log_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1101d0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) disk_log_server","y(3) {state,[]}", "y(4) disk_log_server","y(5) <0.27.0>", "0x00002aaaab110208 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,44}]}, {heap_size,1597}, {total_heap_size,2584}, {links,[<10461.89.0>,<10461.27.0>]}, {memory,21552}, {message_queue_len,0}, {reductions,4613}, {trap_exit,true}]}, {<10461.89.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab440f40 (disk_log:loop/1 + 168)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb8c8f0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) {state,[],[],<0.87.0>,<0.88.0>,567,{arg,latest_log,undefined,\"/var/membase/mnesia", "0x00002aaaabb8c900 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,1597}, {total_heap_size,3194}, {links, [<10461.87.0>,<10461.88.0>,<10461.75.0>, #Port<10461.5368>]}, {memory,26432}, {message_queue_len,0}, {reductions,215983}, {trap_exit,true}]}, {<10461.91.0>, [{registered_name,dets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae1a1c0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,dets_sup},simple_one_for_one,[{child,undefined,dets,{dets,istart_li", "y(4) dets_sup","y(5) <0.27.0>", "0x00002aaaaae1a1f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10461.27.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,1139}, {trap_exit,true}]}, {<10461.92.0>, [{registered_name,dets}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9c710 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) dets_server", "y(3) {state,40996,[<0.27.0>],[]}", "y(4) dets","y(5) <0.27.0>", "0x00002b2847f9c748 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,35}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.27.0>]}, {memory,5640}, {message_queue_len,0}, {reductions,1756}, {trap_exit,true}]}, {<10461.117.0>, [{registered_name,ns_config_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab015c50 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_config_sup},rest_for_one,[{child,undefined,ns_config_log,{ns_con", "y(4) ns_config_sup","y(5) <0.54.0>", "0x00002aaaab015c88 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10461.118.0>,<10461.119.0>,<10461.54.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,580}, {trap_exit,true}]}, {<10461.118.0>, [{registered_name,ns_config_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadc3948 Return addr 0x00002b2847f5d018 (proc_lib:wake_up/3 + 120)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.680>,{state,#Fun,ok},<0.17", "y(3) ns_config_events","y(4) <0.117.0>", "0x00002aaaaadc3978 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5d038 (proc_lib:wake_up/3 + 152)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,24}]}, {heap_size,610}, {total_heap_size,1597}, {links, [<10461.141.0>,<10461.164.0>,<10461.172.0>, <10461.117.0>]}, {memory,13872}, {message_queue_len,0}, {reductions,387411}, {trap_exit,true}]}, {<10461.119.0>, [{registered_name,ns_config}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac169af0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config", "y(3) {config,{full,\"/etc/membase/priv/config\",undefined,ns_config_default},[[{director", "y(4) ns_config","y(5) <0.117.0>", "0x00002aaaac169b28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,46368}, {total_heap_size,92736}, {links,[<10461.117.0>]}, {memory,742648}, {message_queue_len,0}, {reductions,149740}, {trap_exit,true}]}, {<10461.124.0>, [{registered_name,ns_server_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab819868 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_sup},one_for_one,[{child,<0.232.0>,moxi_stats_collector,{", "y(4) ns_server_sup","y(5) <0.54.0>", "0x00002aaaab8198a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,75025}, {total_heap_size,196418}, {links, [<10461.149.0>,<10461.163.0>,<10461.171.0>, <10461.172.0>,<10461.232.0>,<10461.165.0>, <10461.166.0>,<10461.164.0>,<10461.161.0>, <10461.162.0>,<10461.155.0>,<10461.127.0>, <10461.145.0>,<10461.147.0>,<10461.141.0>, <10461.125.0>,<10461.126.0>,<10461.54.0>]}, {memory,1572784}, {message_queue_len,0}, {reductions,87934}, {trap_exit,true}]}, {<10461.125.0>, [{registered_name,ns_log}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae36a30 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) ns_log", "y(3) {state,[{log_entry,{1302,303452,740950},'ns_1@10.1.5.192',ns_node_disco,3,\"Initia", "y(4) ns_log","y(5) <0.124.0>", "0x00002aaaaae36a68 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<10461.50.0>,<10461.124.0>]}, {memory,109040}, {message_queue_len,0}, {reductions,6602}, {trap_exit,false}]}, {<10461.126.0>, [{registered_name,ns_log_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae923e0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []","y(2) []", "y(3) ns_log_events","y(4) <0.124.0>", "0x00002aaaaae92410 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,1597}, {links,[<10461.124.0>]}, {memory,13536}, {message_queue_len,0}, {reductions,88}, {trap_exit,true}]}, {<10461.127.0>, [{registered_name,ns_node_disco_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae93df0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_node_disco_sup},rest_for_one,[{child,<0.136.0>,ns_config_rep,{ns", "y(4) ns_node_disco_sup","y(5) <0.124.0>", "0x00002aaaaae93e28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10461.128.0>,<10461.129.0>,<10461.136.0>, <10461.124.0>]}, {memory,6912}, {message_queue_len,0}, {reductions,829}, {trap_exit,true}]}, {<10461.128.0>, [{registered_name,ns_node_disco_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdc0e50 Return addr 0x00002b2847f5d018 (proc_lib:wake_up/3 + 120)", "y(0) false","y(1) []", "y(2) [{handler,menelaus_event,ns_node_disco_events,{state,ns_node_disco_events,undefin", "y(3) ns_node_disco_events","y(4) <0.127.0>", "0x00002aaaabdc0e80 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5d038 (proc_lib:wake_up/3 + 152)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,44}]}, {heap_size,377}, {total_heap_size,521}, {links,[<10461.127.0>]}, {memory,5144}, {message_queue_len,0}, {reductions,2817}, {trap_exit,true}]}, {<10461.129.0>, [{registered_name,ns_node_disco}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac264010 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_node_disco", "y(3) {state,['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193'],{interval,#Ref<0.0", "y(4) ns_node_disco","y(5) <0.127.0>", "0x00002aaaac264048 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,564}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10461.127.0>,<10461.50.0>]}, {memory,601000}, {message_queue_len,0}, {reductions,42841}, {trap_exit,false}]}, {<10461.136.0>, [{registered_name,ns_config_rep}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaceb68e0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_config_rep","y(3) {state}", "y(4) ns_config_rep","y(5) <0.127.0>", "0x00002aaaaceb6918 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,16}]}, {heap_size,46368}, {total_heap_size,57314}, {links,[<10461.127.0>]}, {memory,459272}, {message_queue_len,0}, {reductions,18821}, {trap_exit,false}]}, {<10461.141.0>, [{registered_name,mb_master}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab59a340 (gen_fsm:loop/7 + 312)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab193850 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mb_master", "y(3) {state,<0.142.0>,'ns_1@10.1.5.192',['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.", "y(4) master","y(5) mb_master", "y(6) <0.124.0>", "0x00002aaaab193890 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,111}]}, {heap_size,1597}, {total_heap_size,1974}, {links, [<10461.118.0>,<10461.124.0>,<10461.142.0>, <10461.50.0>]}, {memory,16672}, {message_queue_len,0}, {reductions,38975}, {trap_exit,false}]}, {<10461.142.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fa9068 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{<0.142.0>,mb_master_sup},one_for_one,[{child,<0.144.0>,ns_tick,{ns_tick,s", "y(4) <0.142.0>","y(5) <0.141.0>", "0x00002b2847fa90a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.143.0>,<10461.144.0>,<10461.141.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,197}, {trap_exit,true}]}, {<10461.143.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab59a340 (gen_fsm:loop/7 + 312)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac319218 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_orchestrator", "y(3) {janitor_state,[\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"],<0.477", "y(4) janitor_running", "y(5) ns_orchestrator","y(6) <0.142.0>", "0x00002aaaac319258 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,31}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10461.142.0>,<10461.4771.0>,<10461.50.0>]}, {memory,601352}, {message_queue_len,0}, {reductions,2346848}, {trap_exit,true}]}, {<10461.144.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab033530 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity","y(2) ns_tick", "y(3) {state,1302304643807}","y(4) ns_tick", "y(5) <0.142.0>", "0x00002aaaab033568 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,72}]}, {heap_size,2584}, {total_heap_size,2961}, {links,[<10461.142.0>,<10461.50.0>]}, {memory,24800}, {message_queue_len,0}, {reductions,46611}, {trap_exit,false}]}, {<10461.145.0>, [{registered_name,ns_heart}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbfc7e8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_heart", "y(3) [{meminfo,<<777 bytes>>},{replication,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9c", "y(4) ns_heart","y(5) <0.124.0>", "0x00002aaaabbfc820 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10461.124.0>,<10461.50.0>]}, {memory,284176}, {message_queue_len,0}, {reductions,1785874}, {trap_exit,false}]}, {<10461.147.0>, [{registered_name,ns_doctor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb61c58 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_doctor", "y(3) {state,{dict,3,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{", "y(4) ns_doctor","y(5) <0.124.0>", "0x00002aaaabb61c90 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10461.124.0>,<10461.50.0>]}, {memory,142488}, {message_queue_len,0}, {reductions,88342}, {trap_exit,false}]}, {<10461.149.0>, [{registered_name,menelaus_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f900b0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,menelaus_sup},one_for_one,[{child,<0.154.0>,menelaus_web_alerts_srv", "y(4) menelaus_sup","y(5) <0.124.0>", "0x00002b2847f900e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,2584}, {total_heap_size,13530}, {links, [<10461.150.0>,<10461.153.0>,<10461.154.0>, <10461.124.0>]}, {memory,109120}, {message_queue_len,0}, {reductions,1870}, {trap_exit,true}]}, {<10461.150.0>, [{registered_name,menelaus_web}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab1430a0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) mochiweb_socket_server", "y(3) {mochiweb_socket_server,8091,#Fun,{local,menelaus_web}", "y(4) menelaus_web","y(5) <0.149.0>", "0x00002aaaab1430d8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,1597}, {total_heap_size,1974}, {links, [<10461.450.0>,<10461.3159.0>,<10461.3204.0>, <10461.2695.0>,<10461.149.0>,<10461.224.0>, #Port<10461.2158>]}, {memory,16792}, {message_queue_len,0}, {reductions,2715}, {trap_exit,true}]}, {<10461.153.0>, [{registered_name,hot_keys_keeper}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847fb8598 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) hot_keys_keeper", "y(3) {state,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",[{\"key_eb13eea", "y(4) hot_keys_keeper","y(5) <0.149.0>", "0x00002b2847fb85d0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,93}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10461.149.0>,<10461.50.0>]}, {memory,142488}, {message_queue_len,0}, {reductions,7390}, {trap_exit,false}]}, {<10461.154.0>, [{registered_name,menelaus_web_alerts_srv}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbe90b8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) menelaus_web_alerts_srv", "y(3) {state,[],[],{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],", "y(4) menelaus_web_alerts_srv", "y(5) <0.149.0>", "0x00002aaaabbe90f0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,680}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10461.50.0>,<10461.149.0>]}, {memory,54920}, {message_queue_len,0}, {reductions,406801}, {trap_exit,false}]}, {<10461.155.0>, [{registered_name,ns_port_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac1aedf8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_port_sup},one_for_one,[{child,<0.222.0>,{moxi,\"./bin/moxi/moxi\",", "y(4) ns_port_sup","y(5) <0.124.0>", "0x00002aaaac1aee30 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10461.222.0>,<10461.428.0>,<10461.124.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,4608}, {trap_exit,true}]}, {<10461.161.0>, [{registered_name,ns_tick_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcb778 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7737>,{state,#Fun,ignored},<", "y(3) ns_tick_event","y(4) <0.124.0>", "0x00002aaaaadcb7a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,257}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.165.0>,<10461.460.0>,<10461.124.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,19571}, {trap_exit,true}]}, {<10461.162.0>, [{registered_name,ns_stats_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f2b9a0 (gen_event:fetch_msg/5 + 88)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab075660 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.7761>,{state,#Fun,ignored},<", "y(3) ns_stats_event","y(4) <0.124.0>", "0x00002aaaab075690 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,501}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10461.166.0>,<10461.461.0>,<10461.124.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,74800}, {trap_exit,true}]}, {<10461.163.0>, [{registered_name,ns_bucket_worker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b284800a988 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) work_queue","y(3) []", "y(4) ns_bucket_worker","y(5) <0.124.0>", "0x00002b284800a9c0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10461.124.0>]}, {memory,8656}, {message_queue_len,0}, {reductions,394}, {trap_exit,false}]}, {<10461.164.0>, [{registered_name,ns_bucket_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0cff70 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_bucket_sup},one_for_one,[{child,<0.446.0>,{per_bucket_sup,\"Failo", "y(4) ns_bucket_sup","y(5) <0.124.0>", "0x00002aaaab0cffa8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10461.124.0>,<10461.446.0>,<10461.118.0>]}, {memory,54960}, {message_queue_len,0}, {reductions,216}, {trap_exit,true}]}, {<10461.165.0>, [{registered_name,system_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadaeeb8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) system_stats_collector", "y(3) {state,#Port<0.2218>,[{cpu_local_ms,1967636310},{cpu_idle_ms,1843305850},{swap_to", "y(4) system_stats_collector", "y(5) <0.124.0>", "0x00002aaaaadaeef0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,305}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10461.124.0>,<10461.161.0>,#Port<10461.2218>]}, {memory,11752}, {message_queue_len,0}, {reductions,69580}, {trap_exit,false}]}, {<10461.166.0>, [{registered_name,'stats_archiver-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff9848 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"@system\"}", "y(4) 'stats_archiver-@system'", "y(5) <0.124.0>", "0x00002b2847ff9880 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,158}]}, {heap_size,987}, {total_heap_size,3571}, {links,[<10461.124.0>,<10461.162.0>,<10461.50.0>]}, {memory,29408}, {message_queue_len,0}, {reductions,6062062}, {trap_exit,false}]}, {<10461.171.0>, [{registered_name,'stats_reader-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff6648 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"@system\"}", "y(4) 'stats_reader-@system'", "y(5) <0.124.0>", "0x00002b2847ff6680 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.124.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,26}, {trap_exit,false}]}, {<10461.172.0>, [{registered_name,ns_moxi_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc0ec38 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_moxi_sup},one_for_one,[{child,<0.441.0>,{\"FailoverNodesTests-4a4", "y(4) ns_moxi_sup","y(5) <0.124.0>", "0x00002aaaabc0ec70 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10461.124.0>,<10461.441.0>,<10461.118.0>]}, {memory,142528}, {message_queue_len,0}, {reductions,1343}, {trap_exit,true}]}, {<10461.222.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb1968 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi,5000,{1302,303457,478273},<0.223.0>}", "y(4) <0.222.0>","y(5) <0.155.0>", "0x00002aaaaadb19a0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10461.155.0>,<10461.223.0>]}, {memory,21472}, {message_queue_len,0}, {reductions,150}, {trap_exit,true}]}, {<10461.223.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab06b4c8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2338>,moxi,{[\"2011-04-08 15:57:37: (cproxy_config.c.325) env: MOXI", "y(4) <0.223.0>","y(5) <0.222.0>", "0x00002aaaab06b500 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10461.222.0>,#Port<10461.2338>]}, {memory,26352}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10461.224.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab5fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac0ec818 Return addr 0x00002aaaab5f9b78 (menelaus_web:loop/3 + 26376)", "y(0) {struct,[{buckets,[]}]}", "y(1) {struct,[{buckets,[]}]}", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBuck", "y(3) {mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(4) #Fun", "0x00002aaaac0ec848 Return addr 0x00002aaaab5d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(5) Catch 0x00002aaaab5f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaac0ec880 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2342>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "0x00002aaaac0ec8a8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,28657}, {total_heap_size,28657}, {links,[<10461.150.0>,#Port<10461.2342>]}, {memory,230200}, {message_queue_len,0}, {reductions,7084}, {trap_exit,false}]}, {<10461.232.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ffab68 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi_stats_collector,5000,{1302,303457,921207},<0.233.0>}", "y(4) <0.232.0>","y(5) <0.124.0>", "0x00002b2847ffaba0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.124.0>,<10461.233.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<10461.233.0>, [{registered_name,moxi_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdc0238 Return addr 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(0) []","y(1) infinity", "y(2) moxi_stats_collector", "y(3) {state,#Port<0.2357>,{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[", "y(4) <0.233.0>","y(5) <0.232.0>", "0x00002aaaabdc0270 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) Catch 0x00002aaaaabfc400 (gen_server:init_it/6 + 392)", "y(1) moxi_stats_collector","y(2) []", "y(3) moxi_stats_collector", "y(4) {local,moxi_stats_collector}", "y(5) <0.232.0>","y(6) <0.232.0>", "0x00002aaaabdc02b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,15}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [#Port<10461.2603>,<10461.232.0>,#Port<10461.2357>]}, {memory,263544}, {message_queue_len,0}, {reductions,11780463}, {trap_exit,false}]}, {<10461.428.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0372d8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,memcached,5000,{1302,303773,930402},<0.429.0>}", "y(4) <0.428.0>","y(5) <0.155.0>", "0x00002aaaab037310 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10461.155.0>,<10461.429.0>]}, {memory,8696}, {message_queue_len,0}, {reductions,55}, {trap_exit,true}]}, {<10461.429.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaaea9fb8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2527>,memcached,{[empty],[empty,empty]},undefined,[],0}", "y(4) <0.429.0>","y(5) <0.428.0>", "0x00002aaaaaea9ff0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10461.428.0>,#Port<10461.2527>]}, {memory,8696}, {message_queue_len,0}, {reductions,58}, {trap_exit,true}]}, {<10461.441.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0a4038 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2539>,moxi,{[\"ERROR: could not contact REST server(s): http://127.", "y(4) <0.441.0>","y(5) <0.172.0>", "0x00002aaaab0a4070 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10461.172.0>,#Port<10461.2539>]}, {memory,21472}, {message_queue_len,0}, {reductions,356}, {trap_exit,true}]}, {<10461.446.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaab3fe9a0 (single_bucket_sup:top_loop/3 + 72)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847f9ce80 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "y(2) <0.447.0>","y(3) <0.164.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.164.0>,<10461.447.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,31}, {trap_exit,true}]}, {<10461.447.0>, [{registered_name, 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab03b7e8 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95", "y(4) 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.446.0>", "0x00002aaaab03b820 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,1220}, {links, [<10461.460.0>,<10461.461.0>,<10461.462.0>, <10461.448.0>,<10461.459.0>,<10461.446.0>]}, {memory,10720}, {message_queue_len,0}, {reductions,869}, {trap_exit,true}]}, {<10461.448.0>, [{registered_name, 'ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab18d478 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) ns_memcached", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",#Port<0.2563>}", "y(4) <0.448.0>","y(5) <0.447.0>", "0x00002aaaab18d4b0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,72}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10461.50.0>,<10461.447.0>,#Port<10461.2563>]}, {memory,284216}, {message_queue_len,0}, {reductions,31527810}, {trap_exit,true}]}, {<10461.450.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab5fd5d8 (menelaus_web:handle_streaming/4 + 392)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaac3d6658 Return addr 0x00002aaaab65fd20 (menelaus_web_buckets:checking_bucket_ac", "y(0) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(1) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsS", "y(3) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) #Fun", "0x00002aaaac3d6688 Return addr 0x00002aaaab5f9a28 (menelaus_web:loop/3 + 26040)", "y(0) #Ref<0.0.0.7709>", "y(1) Catch 0x00002aaaab65fd80 (menelaus_web_buckets:checking_bucket_access/4 + 616)", "y(2) []", "y(3) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) []", "0x00002aaaac3d66b8 Return addr 0x00002aaaab5d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(5) Catch 0x00002aaaab5f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaac3d66f0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2574>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "0x00002aaaac3d6718 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,17711}, {total_heap_size,92736}, {links,[<10461.150.0>,#Port<10461.2574>]}, {memory,742832}, {message_queue_len,0}, {reductions,1482307}, {trap_exit,false}]}, {<10461.459.0>, [{registered_name, 'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0a2158 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(4) 'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.447.0>", "0x00002aaaab0a2190 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10461.447.0>]}, {memory,54880}, {message_queue_len,0}, {reductions,1281}, {trap_exit,true}]}, {<10461.460.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab0f79c0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_collector", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\",{[157,8053131,27", "y(4) <0.460.0>","y(5) <0.447.0>", "0x00002aaaab0f79f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,33}]}, {heap_size,6765}, {total_heap_size,24476}, {links,[<10461.447.0>,<10461.161.0>]}, {memory,196608}, {message_queue_len,0}, {reductions,20247767}, {trap_exit,false}]}, {<10461.461.0>, [{registered_name, 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabbd6c80 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"}", "y(4) 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.447.0>", "0x00002aaaabbd6cb8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,414}]}, {heap_size,6765}, {total_heap_size,53133}, {links,[<10461.162.0>,<10461.447.0>,<10461.50.0>]}, {memory,425904}, {message_queue_len,0}, {reductions,4736583}, {trap_exit,false}]}, {<10461.462.0>, [{registered_name, 'stats_reader-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaad5029f0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"}", "y(4) 'stats_reader-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(5) <0.447.0>", "0x00002aaaad502a28 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,196418}, {total_heap_size,271443}, {links,[<10461.447.0>]}, {memory,2172304}, {message_queue_len,0}, {reductions,595219}, {trap_exit,false}]}, {<10461.584.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabf4ba8 (application_master:main_loop/2 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadb2ca0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []", "y(1) {state,<0.585.0>,{appl_data,inets,[inets_sup,httpc_manager],undefined,{inets_app,", "y(2) <0.7.0>", "0x00002aaaaadb2cc0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.7.0>,<10461.585.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<10461.585.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0x00002aaaaabf7230 (application_master:loop_it/4 + 80)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab067eb0 Return addr 0x0000000000816eb8 ()", "y(0) []","y(1) inets_app", "y(2) <0.586.0>","y(3) <0.584.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.584.0>,<10461.586.0>]}, {memory,2560}, {message_queue_len,0}, {reductions,42}, {trap_exit,true}]}, {<10461.586.0>, [{registered_name,inets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae99a88 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,inets_sup},one_for_one,[{child,<0.593.0>,tftp_sup,{tftp_sup,start_l", "y(4) inets_sup","y(5) <0.585.0>", "0x00002aaaaae99ac0 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10461.587.0>,<10461.592.0>,<10461.593.0>, <10461.588.0>,<10461.585.0>]}, {memory,6952}, {message_queue_len,0}, {reductions,320}, {trap_exit,true}]}, {<10461.587.0>, [{registered_name,ftp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae910c0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ftp_sup},simple_one_for_one,[{child,undefined,undefined,{ftp,start_", "y(4) ftp_sup","y(5) <0.586.0>", "0x00002aaaaae910f8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.586.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,54}, {trap_exit,true}]}, {<10461.588.0>, [{registered_name,httpc_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadbefb0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpc_sup},one_for_one,[{child,<0.591.0>,httpc_handler_sup,{httpc_h", "y(4) httpc_sup","y(5) <0.586.0>", "0x00002aaaaadbefe8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10461.589.0>,<10461.591.0>,<10461.586.0>]}, {memory,5720}, {message_queue_len,0}, {reductions,175}, {trap_exit,true}]}, {<10461.589.0>, [{registered_name,httpc_profile_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae43940 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpc_profile_sup},one_for_one,[{child,<0.590.0>,httpc_manager,{htt", "y(4) httpc_profile_sup","y(5) <0.588.0>", "0x00002aaaaae43978 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.588.0>,<10461.590.0>]}, {memory,2664}, {message_queue_len,0}, {reductions,124}, {trap_exit,true}]}, {<10461.590.0>, [{registered_name,httpc_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae0c9b0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) httpc_manager", "y(3) {state,[],4374581,{undefined,4378678},httpc_manager_session_db,httpc_manager,{opt", "y(4) httpc_manager","y(5) <0.589.0>", "0x00002aaaaae0c9e8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,2584}, {total_heap_size,6765}, {links,[<10461.589.0>]}, {memory,54880}, {message_queue_len,0}, {reductions,615}, {trap_exit,true}]}, {<10461.591.0>, [{registered_name,httpc_handler_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabcc0820 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpc_handler_sup},simple_one_for_one,[{child,undefined,undefined,{", "y(4) httpc_handler_sup","y(5) <0.588.0>", "0x00002aaaabcc0858 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10461.588.0>]}, {memory,54880}, {message_queue_len,0}, {reductions,177}, {trap_exit,true}]}, {<10461.592.0>, [{registered_name,httpd_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab144b10 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,httpd_sup},one_for_one,[],{dict,0,16,16,8,80,48,{[],[],[],[],[],[],", "y(4) httpd_sup","y(5) <0.586.0>", "0x00002aaaab144b48 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.586.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<10461.593.0>, [{registered_name,tftp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaabfcc90 (gen_server:loop/6 + 288)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaae55628 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,tftp_sup},one_for_one,[],{dict,0,16,16,8,80,48,{[],[],[],[],[],[],[", "y(4) tftp_sup","y(5) <0.586.0>", "0x00002aaaaae55660 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.586.0>]}, {memory,2624}, {message_queue_len,0}, {reductions,46}, {trap_exit,true}]}, {<10461.604.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_accept,6}}, {backtrace, ["Program counter: 0x00002aaaabedccd8 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabdc4028 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,4237,8565,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.192'", "y(6) {net_address,{{10,1,5,193},38457},\"10.1.5.193\",tcp,inet}", "y(7) #Port<0.3452>", "y(8) 'ns_1@10.1.5.193'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,9}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10461.61.0>,#Port<10461.3452>]}, {memory,16488}, {message_queue_len,0}, {reductions,1770}, {trap_exit,false}]}, {<10461.643.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_setup,6}}, {backtrace, ["Program counter: 0x00002aaaabedccd8 (dist_util:con_loop/9 + 144)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab077528 Return addr 0x0000000000816eb8 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,4338,9024,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.192'", "y(6) {net_address,{{10,1,5,178},21100},\"10.1.5.178\",tcp,inet}", "y(7) #Port<0.3471>", "y(8) 'ns_1@10.1.5.178'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10461.61.0>,#Port<10461.3471>]}, {memory,11608}, {message_queue_len,0}, {reductions,1735}, {trap_exit,false}]}, {<10461.2695.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaab5fb130 (menelaus_web:handle_pool_info_wait/6 + 280)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabc53eb0 Return addr 0x00002aaaab5fabb0 (menelaus_web:check_and_handle_pool_info", "y(0) \"129590282\"","y(1) \"129590282\"", "y(2) 20000","y(3) \"10.1.5.192\"", "y(4) {\"Administrator\",\"password\"}", "y(5) \"default\"", "y(6) {mochiweb_request,#Port<0.5122>,'GET',\"/pools/default?waitChange=20000&etag=12959", "0x00002aaaabc53ef0 Return addr 0x00002aaaab5f9b08 (menelaus_web:loop/3 + 26264)", "y(0) Catch 0x00002aaaab5fac08 (menelaus_web:check_and_handle_pool_info/2 + 336)", "y(1) []","y(2) <0.3203.0>", "0x00002aaaabc53f10 Return addr 0x00002aaaab5d1be8 (mochiweb_http:headers/5 + 1360)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.5122>,'GET',\"/pools/default?waitChange=20000&etag=12959", "y(5) Catch 0x00002aaaab5f9b98 (menelaus_web:loop/3 + 26408)", "0x00002aaaabc53f48 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.5122>,'GET',\"/pools/default?waitChange=20000&etag=12959", "0x00002aaaabc53f70 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,17711}, {total_heap_size,46368}, {links, [<10461.150.0>,<10461.3203.0>,#Port<10461.5122>]}, {memory,371928}, {message_queue_len,0}, {reductions,938540}, {trap_exit,false}]}, {<10461.3159.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f16000 (prim_inet:recv0/3 + 224)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaab13feb8 Return addr 0x00002aaaab5d1310 (mochiweb_http:request/2 + 120)", "y(0) 20294","y(1) #Port<0.5435>", "0x00002aaaab13fed0 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) #Port<0.5435>", "0x00002aaaab13fee8 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,10946}, {total_heap_size,28657}, {links,[<10461.150.0>,#Port<10461.5435>]}, {memory,230056}, {message_queue_len,0}, {reductions,30579}, {trap_exit,false}]}, {<10461.3203.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002aaaab500158 (diag_handler:'-arm_timeout/2-fun-0-'/3 + 64)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002b2847ff6dc0 Return addr 0x0000000000816eb8 ()", "y(0) <0.2695.0>", "y(1) #Fun", "y(2) 23000"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.2695.0>]}, {memory,2520}, {message_queue_len,0}, {reductions,2}, {trap_exit,false}]}, {<10461.3204.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002b2847f14ba0 (prim_inet:accept0/2 + 184)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaaadcc600 Return addr 0x00002aaaaaf7e198 (inet_tcp:accept/1 + 40)", "y(0) 57562","y(1) #Port<0.2158>", "0x00002aaaaadcc618 Return addr 0x00002aaaab5dfe88 (mochiweb_socket_server:acceptor_loop/1 ", "y(0) []", "0x00002aaaaadcc628 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) #Fun", "y(1) <0.150.0>", "y(2) Catch 0x00002aaaab5dfe88 (mochiweb_socket_server:acceptor_loop/1 + 160)", "0x00002aaaaadcc648 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10461.150.0>]}, {memory,2696}, {message_queue_len,0}, {reductions,38}, {trap_exit,false}]}, {<10461.4771.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x00002aaaaaee3ea8 (timer:sleep/1 + 40)", "CP: 0x0000000000000000 (invalid)","arity = 0", "0x00002aaaabb1c940 Return addr 0x00002aaaaba08cf0 (ns_janitor:wait_for_memcached/3 + 184)", "y(0) 1000", "0x00002aaaabb1c950 Return addr 0x00002aaaaba06f98 (ns_janitor:cleanup/1 + 992)", "y(0) ['ns_1@10.1.5.193','ns_1@10.1.5.178']", "y(1) 4", "y(2) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "0x00002aaaabb1c970 Return addr 0x00002b2847f5cf38 (proc_lib:init_p_do_apply/3 + 56)", "y(0) [['ns_1@10.1.5.192',undefined],['ns_1@10.1.5.192',undefined],['ns_1@10.1.5.192',u", "y(1) ['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']", "y(2) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "0x00002aaaabb1c990 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002b2847f5cf58 (proc_lib:init_p_do_apply/3 + 88)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<10461.143.0>]}, {memory,142448}, {message_queue_len,0}, {reductions,194}, {trap_exit,false}]}, {<10461.6292.0>, [{registered_name,[]}, {status,running}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0x00002b2847e998d8 (unknown function)", "CP: 0x00002aaaab4fe068 (diag_handler:grab_process_info/1 + 64)", "0x00002aaaaddbae70 Return addr 0x00002aaaab500610 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []", "0x00002aaaaddbae80 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) []","y(1) <0.6292.0>", "0x00002aaaaddbae98 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.4771.0>", "0x00002aaaaddbaeb0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.3204.0>", "0x00002aaaaddbaec8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.3203.0>", "0x00002aaaaddbaee0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.3159.0>", "0x00002aaaaddbaef8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.2695.0>", "0x00002aaaaddbaf10 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_setup,6}},", "y(1) <0.643.0>", "0x00002aaaaddbaf28 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_accept,6}}", "y(1) <0.604.0>", "0x00002aaaaddbaf40 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,tftp_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.593.0>", "0x00002aaaaddbaf58 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpd_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.592.0>", "0x00002aaaaddbaf70 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_handler_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.591.0>", "0x00002aaaaddbaf88 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_manager},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.590.0>", "0x00002aaaaddbafa0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_profile_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.589.0>", "0x00002aaaaddbafb8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,httpc_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.588.0>", "0x00002aaaaddbafd0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ftp_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.587.0>", "0x00002aaaaddbafe8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.586.0>", "0x00002aaaaddbb000 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.585.0>", "0x00002aaaaddbb018 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.584.0>", "0x00002aaaaddbb030 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_reader-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) <0.462.0>", "0x00002aaaaddbb048 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b9", "y(1) <0.461.0>", "0x00002aaaaddbb060 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.460.0>", "0x00002aaaaddbb078 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95820", "y(1) <0.459.0>", "0x00002aaaaddbb090 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.450.0>", "0x00002aaaaddbb0a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) <0.448.0>", "0x00002aaaaddbb0c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-5", "y(1) <0.447.0>", "0x00002aaaaddbb0d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.446.0>", "0x00002aaaaddbb0f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.441.0>", "0x00002aaaaddbb108 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.429.0>", "0x00002aaaaddbb120 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.428.0>", "0x00002aaaaddbb138 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,moxi_stats_collector},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.233.0>", "0x00002aaaaddbb150 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.232.0>", "0x00002aaaaddbb168 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.224.0>", "0x00002aaaaddbb180 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.223.0>", "0x00002aaaaddbb198 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.222.0>", "0x00002aaaaddbb1b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_moxi_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.172.0>", "0x00002aaaaddbb1c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_reader-@system'},{status,waiting},{initial_call,{proc_li", "y(1) <0.171.0>", "0x00002aaaaddbb1e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,'stats_archiver-@system'},{status,waiting},{initial_call,{proc_", "y(1) <0.166.0>", "0x00002aaaaddbb1f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,system_stats_collector},{status,waiting},{initial_call,{proc_li", "y(1) <0.165.0>", "0x00002aaaaddbb210 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.164.0>", "0x00002aaaaddbb228 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_bucket_worker},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.163.0>", "0x00002aaaaddbb240 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_stats_event},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.162.0>", "0x00002aaaaddbb258 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_tick_event},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.161.0>", "0x00002aaaaddbb270 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_port_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.155.0>", "0x00002aaaaddbb288 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web_alerts_srv},{status,waiting},{initial_call,{proc_l", "y(1) <0.154.0>", "0x00002aaaaddbb2a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,hot_keys_keeper},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.153.0>", "0x00002aaaaddbb2b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_web},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.150.0>", "0x00002aaaaddbb2d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,menelaus_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.149.0>", "0x00002aaaaddbb2e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_doctor},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.147.0>", "0x00002aaaaddbb300 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_heart},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.145.0>", "0x00002aaaaddbb318 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.144.0>", "0x00002aaaaddbb330 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.143.0>", "0x00002aaaaddbb348 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.142.0>", "0x00002aaaaddbb360 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_master},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.141.0>", "0x00002aaaaddbb378 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_rep},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.136.0>", "0x00002aaaaddbb390 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.129.0>", "0x00002aaaaddbb3a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_events},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.128.0>", "0x00002aaaaddbb3c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_node_disco_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.127.0>", "0x00002aaaaddbb3d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log_events},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.126.0>", "0x00002aaaaddbb3f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_log},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.125.0>", "0x00002aaaaddbb408 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.124.0>", "0x00002aaaaddbb420 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.119.0>", "0x00002aaaaddbb438 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.118.0>", "0x00002aaaaddbb450 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_config_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.117.0>", "0x00002aaaaddbb468 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.92.0>", "0x00002aaaaddbb480 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.91.0>", "0x00002aaaaddbb498 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.89.0>", "0x00002aaaaddbb4b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_server},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.88.0>", "0x00002aaaaddbb4c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disk_log_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.87.0>", "0x00002aaaaddbb4e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_late_loader},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.83.0>", "0x00002aaaaddbb4f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_controller},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.82.0>", "0x00002aaaaddbb510 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_snmp_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.81.0>", "0x00002aaaaddbb528 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_checkpoint_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.80.0>", "0x00002aaaaddbb540 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_tm},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.79.0>", "0x00002aaaaddbb558 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_recover},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.78.0>", "0x00002aaaaddbb570 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_locker},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.77.0>", "0x00002aaaaddbb588 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_subscr},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.76.0>", "0x00002aaaaddbb5a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_monitor},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.75.0>", "0x00002aaaaddbb5b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_kernel_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.74.0>", "0x00002aaaaddbb5d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_event},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.73.0>", "0x00002aaaaddbb5e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.72.0>", "0x00002aaaaddbb600 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.71.0>", "0x00002aaaaddbb618 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.70.0>", "0x00002aaaaddbb630 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.67.0>", "0x00002aaaaddbb648 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.66.0>", "0x00002aaaaddbb660 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,mb_mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.65.0>", "0x00002aaaaddbb678 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_cluster},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.64.0>", "0x00002aaaaddbb690 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{net_kernel,ticker,2}},{back", "y(1) <0.63.0>", "0x00002aaaaddbb6a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,accept_loop,2", "y(1) <0.62.0>", "0x00002aaaaddbb6c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_kernel},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.61.0>", "0x00002aaaaddbb6d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,auth},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.60.0>", "0x00002aaaaddbb6f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_epmd},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.59.0>", "0x00002aaaaddbb708 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,net_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.58.0>", "0x00002aaaaddbb720 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native},{status,waiting},{initial_call,{inet_getho", "y(1) <0.57.0>", "0x00002aaaaddbb738 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_gethost_native_sup},{status,waiting},{initial_call,{proc_l", "y(1) <0.56.0>", "0x00002aaaaddbb750 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,dist_manager},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.55.0>", "0x00002aaaaddbb768 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,ns_server_cluster_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.54.0>", "0x00002aaaaddbb780 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.53.0>", "0x00002aaaaddbb798 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.52.0>", "0x00002aaaaddbb7b0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,timer_server},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.50.0>", "0x00002aaaaddbb7c8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.49.0>", "0x00002aaaaddbb7e0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.48.0>", "0x00002aaaaddbb7f8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,cpu_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.47.0>", "0x00002aaaaddbb810 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.46.0>", "0x00002aaaaddbb828 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,memsup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.45.0>", "0x00002aaaaddbb840 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,disksup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.44.0>", "0x00002aaaaddbb858 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,os_mon_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.43.0>", "0x00002aaaaddbb870 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.42.0>", "0x00002aaaaddbb888 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.41.0>", "0x00002aaaaddbb8a0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,release_handler},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.38.0>", "0x00002aaaaddbb8b8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,overload},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.37.0>", "0x00002aaaaddbb8d0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,alarm_handler},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.36.0>", "0x00002aaaaddbb8e8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_safe_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.35.0>", "0x00002aaaaddbb900 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,sasl_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.34.0>", "0x00002aaaaddbb918 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.33.0>", "0x00002aaaaddbb930 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.32.0>", "0x00002aaaaddbb948 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_safe_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.27.0>", "0x00002aaaaddbb960 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.26.0>", "0x00002aaaaddbb978 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,user},{status,waiting},{initial_call,{user,server,2}},{backtrac", "y(1) <0.25.0>", "0x00002aaaaddbb990 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.24.0>", "0x00002aaaaddbb9a8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error},{status,waiting},{initial_call,{standard_error,", "y(1) <0.23.0>", "0x00002aaaaddbb9c0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,standard_error_sup},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.22.0>", "0x00002aaaaddbb9d8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,code_server},{status,waiting},{initial_call,{erlang,apply,2}},{", "y(1) <0.21.0>", "0x00002aaaaddbb9f0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,file_server_2},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.20.0>", "0x00002aaaaddbba08 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_group},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.19.0>", "0x00002aaaaddbba20 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,inet_db},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.17.0>", "0x00002aaaaddbba38 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.16.0>", "0x00002aaaaddbba50 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.15.0>", "0x00002aaaaddbba68 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.14.0>", "0x00002aaaaddbba80 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,global_name_server},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.13.0>", "0x00002aaaaddbba98 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,rex},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backt", "y(1) <0.12.0>", "0x00002aaaaddbbab0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,kernel_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.11.0>", "0x00002aaaaddbbac8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.10.0>", "0x00002aaaaddbbae0 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.9.0>", "0x00002aaaaddbbaf8 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,application_controller},{status,waiting},{initial_call,{erlang,", "y(1) <0.7.0>", "0x00002aaaaddbbb10 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,error_logger},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.6.0>", "0x00002aaaaddbbb28 Return addr 0x00002aaaab500650 (diag_handler:'-do_diag_per_node/0-lc$^0", "y(0) [{registered_name,erl_prim_loader},{status,waiting},{initial_call,{erlang,apply,2", "y(1) <0.3.0>", "0x00002aaaaddbbb40 Return addr 0x00002aaaab4fe278 (diag_handler:do_diag_per_node/0 + 224)", "y(0) [{registered_name,init},{status,waiting},{initial_call,{otp_ring0,start,2}},{back", "y(1) <0.0.0>", "0x00002aaaaddbbb58 Return addr 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 20", "y(0) []","y(1) []", "y(2) [{version,[{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"}", "y(3) [{alerts,[{email,[]},{email_alerts,false},{email_server,[{user,undefined},{pass,'", "y(4) [\"bucket_engine_1.6.5_10_g3c82cbf-Linux.x86_64.tar.gz\",\"curl-7.21.1-w64_patched.t", "y(5) [{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"},{inets,\"5", "0x00002aaaaddbbb90 Return addr 0x0000000000816eb8 ()", "y(0) Catch 0x00002aaaaac6bb10 (rpc:'-handle_call_call/6-fun-0-'/5 + 208)", "y(1) []","y(2) []","y(3) []", "y(4) <0.12.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,317811}, {total_heap_size,832040}, {links,[]}, {memory,6657008}, {message_queue_len,0}, {reductions,720569}, {trap_exit,false}]}]}, {memory,{2058928128,1594331136,{<10461.462.0>,2172304}}}, {disk, [{"/",13140720,42}, {"/boot",101086,13}, {"/dev/shm",1005336,0}]}]}, {'ns_1@10.1.5.178', [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {manifest, ["bucket_engine_1.6.5_10_g3c82cbf-Linux.i686.tar.gz", "curl-7.21.1-w64_patched.tar.gz", "ep-engine_1.6.5.3_143_g366c22e-Linux.i686.tar.gz", "google-perftools-1.7","google-perftools-1.7.tar.gz", "libconflate_1.6.5_1_g8a42194-Linux.i686.tar.gz", "libevent-2.0.7-rc.tar.gz", "libmemcached-0.41_trond-norbye_mingw32-revno895.tar.gz", "libvbucket_1.6.4_1_g5cb588e-Linux.i686.tar.gz", "membase-cli_1.6.5_2_gb9c8a7d-Linux.i686.tar.gz", "memcached_1.4.4_430_g6235145-Linux.i686.tar.gz", "moxi_1.6.5.1_16_g8128746-Linux.i686.tar.gz", "ns_server_1.6.5.3_291_gba5fe27.tar.gz", "portsigar_-Linux.i686.tar.gz", "sigar_sigar_1.6.4_402_g7809620-Linux.i686.tar.gz", "vbucketmigrator_1.7.0r_4_g8428c29-Linux.i686.tar.gz", "wallace_1.7.0r-100-g85ee8f5-Linux.i686-201104060841"]}, {config, [{alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,'filtered-out'}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up, server_joined,server_left,bucket_created, bucket_deleted,bucket_auth_failed]}]}, {buckets, [{'_vclock',[{'ns_1@10.1.5.192',{11,63469523235}}]}, {configs, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers, ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178']}, {map, [['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}]}]}, {memory_quota,1570}, {nodes_wanted, [{'_vclock',[{'ns_1@10.1.5.192',{2,63469523039}}]}, 'ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193']}, {otp, [{'_vclock',[{'ns_1@10.1.5.193',{1,63469523027}}]}, {cookie,lulxzmudqfmlvwvs}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {rebalance_status, {none, <<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>}}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds, [{'_vclock',[{'ns_1@10.1.5.192',{1,63469522657}}]}, {creds, [{"Administrator",[{password,'filtered-out'}]}]}]}, {{node,'ns_1@10.1.5.178',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.178',membership},inactiveFailed}, {{node,'ns_1@10.1.5.178',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.178',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.178',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.178',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.192',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.192',membership},active}, {{node,'ns_1@10.1.5.192',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.192',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.192',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.192',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.193',isasl}, [{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.193',membership},inactiveFailed}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached, [{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.193',moxi}, [{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}]}, {basic_info, [{version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,1114}, {memory_data, {2075373568,1765871616,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41}, {"/boot",101086,13}, {"/dev/shm",1013364,0}]}]}, {processes, [{<10509.0.0>, [{registered_name,init}, {status,waiting}, {initial_call,{otp_ring0,start,2}}, {backtrace, ["Program counter: 0xb7c79408 (init:loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6db6518 Return addr 0x081f9e74 ()", "y(0) {state,[{'-root',[<<37 bytes>>]},{'-progname',[<<3 bytes>>]},{'-home',[<<12 bytes"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,87}]}, {heap_size,2584}, {total_heap_size,9349}, {links,[<10509.6.0>,<10509.7.0>,<10509.3.0>]}, {memory,37776}, {message_queue_len,0}, {reductions,47055}, {trap_exit,true}]}, {<10509.3.0>, [{registered_name,erl_prim_loader}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7e7e860 (erl_prim_loader:loop/3 + 92)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6fc5bfc Return addr 0x081f9e74 ()", "y(0) []", "y(1) [\"/opt/membase/1.7.0r/bin/ns_server/deps/menelaus/deps/mochiweb/ebin\",\"/opt/memba", "y(2) <0.2.0>", "y(3) {state,efile,[],none,#Port<0.2>,infinity,undefined,true,{prim_state,false,undefin", "y(4) infinity"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,267}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[#Port<10509.2>,<10509.0.0>]}, {memory,54480}, {message_queue_len,0}, {reductions,887120}, {trap_exit,true}]}, {<10509.6.0>, [{registered_name,error_logger}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb768efc4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,ns_log_mf_h,false,{state,\"/var/membase/logs\",10485760,10,616544,1,{file", "y(3) error_logger","y(4) <0.2.0>", "0xb768efdc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,30}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<10509.0.0>,<10509.25.0>,#Port<10509.965>]}, {memory,33888}, {message_queue_len,0}, {reductions,220884}, {trap_exit,true}]}, {<10509.7.0>, [{registered_name,application_controller}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d640f8 Return addr 0x081f9e74 ()", "y(0) []","y(1) infinity", "y(2) application_controller", "y(3) {state,[],[],[],[{ns_server,<0.52.0>},{mnesia,<0.70.0>},{os_mon,<0.41.0>},{sasl,<", "y(4) application_controller", "y(5) <0.2.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,4181}, {total_heap_size,21892}, {links, [<10509.41.0>,<10509.52.0>,<10509.70.0>,<10509.9.0>, <10509.32.0>,<10509.0.0>]}, {memory,88068}, {message_queue_len,0}, {reductions,31293}, {trap_exit,true}]}, {<10509.9.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce8324 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.10.0>,{appl_data,kernel,[application_controller,erl_reply,auth,boot_ser", "y(2) <0.7.0>", "0xb7ce8334 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10509.7.0>,<10509.10.0>]}, {memory,3436}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<10509.10.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ceb2e0 Return addr 0x081f9e74 ()", "y(0) []","y(1) kernel","y(2) <0.11.0>", "y(3) <0.9.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.9.0>,<10509.11.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,72}, {trap_exit,true}]}, {<10509.11.0>, [{registered_name,kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d07ea0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_sup},one_for_all,[{child,<0.58.0>,net_sup_dynamic,{erl_distr", "y(4) kernel_sup","y(5) <0.10.0>", "0xb7d07ebc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10509.22.0>,<10509.26.0>,<10509.27.0>,<10509.58.0>, <10509.24.0>,<10509.17.0>,<10509.20.0>,<10509.21.0>, <10509.19.0>,<10509.12.0>,<10509.13.0>,<10509.10.0>]}, {memory,6076}, {message_queue_len,0}, {reductions,1451}, {trap_exit,true}]}, {<10509.12.0>, [{registered_name,rex}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb668ccfc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) rpc", "y(3) {1,{<0.1162.0>,{<10507.1193.0>,{#Ref<10507.0.0.24792>,'ns_1@10.1.5.178'}},nil,nil", "y(4) rex","y(5) <0.11.0>", "0xb668cd18 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,196418}, {total_heap_size,196418}, {links,[<10509.11.0>]}, {memory,786176}, {message_queue_len,0}, {reductions,2997}, {trap_exit,true}]}, {<10509.13.0>, [{registered_name,global_name_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7772fb8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) global", "y(3) {state,true,['ns_1@10.1.5.192','ns_1@10.1.5.193'],['ns_1@10.1.5.193','ns_1@10.1.5", "y(4) global_name_server","y(5) <0.11.0>", "0xb7772fd4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,987}, {total_heap_size,1364}, {links, [<10509.14.0>,<10509.16.0>,<10509.15.0>,<10509.11.0>]}, {memory,6068}, {message_queue_len,0}, {reductions,1945}, {trap_exit,true}]}, {<10509.14.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7f31094 (global:loop_the_locker/1 + 588)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cec254 Return addr 0xb7f30e28 (global:init_the_locker/1 + 192)", "y(0) {multi,[],[],['ns_1@10.1.5.192','ns_1@10.1.5.193'],'ns_1@10.1.5.193',false,false}", "y(1) infinity", "0xb7cec260 Return addr 0x081f9e74 ()"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10509.13.0>]}, {memory,5796}, {message_queue_len,0}, {reductions,547}, {trap_exit,true}]}, {<10509.15.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7f35140 (global:collect_deletions/2 + 76)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf1ee4 Return addr 0xb7f350c0 (global:loop_the_deleter/1 + 36)", "y(0) infinity","y(1) []", "y(2) <0.13.0>", "0xb7cf1ef4 Return addr 0x081f9e74 ()", "y(0) <0.13.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10509.13.0>]}, {memory,3356}, {message_queue_len,0}, {reductions,227}, {trap_exit,false}]}, {<10509.16.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7f35260 (global:loop_the_registrar/0 + 12)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d59d58 Return addr 0x081f9e74 ()", "y(0) []"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10509.13.0>]}, {memory,4288}, {message_queue_len,0}, {reductions,243}, {trap_exit,false}]}, {<10509.17.0>, [{registered_name,inet_db}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10158 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) inet_db", "y(3) {state,inet_db,inet_cache,inet_hosts_byname,inet_hosts_byaddr,inet_hosts_file_byn", "y(4) inet_db","y(5) <0.11.0>", "0xb7d10174 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10509.11.0>]}, {memory,10736}, {message_queue_len,0}, {reductions,1444}, {trap_exit,true}]}, {<10509.19.0>, [{registered_name,global_group}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce7748 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) global_group", "y(3) {state,no_conf,true,[],[],[],[],[],'nonode@nohost',[],normal,normal}", "y(4) global_group","y(5) <0.11.0>", "0xb7ce7764 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.11.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,84}, {trap_exit,true}]}, {<10509.20.0>, [{registered_name,file_server_2}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6e10e08 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) file_server","y(3) #Port<0.223>", "y(4) file_server_2","y(5) <0.11.0>", "0xb6e10e24 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,154}]}, {heap_size,6765}, {total_heap_size,24476}, {links,[#Port<10509.223>,<10509.11.0>]}, {memory,98324}, {message_queue_len,0}, {reductions,601338}, {trap_exit,true}]}, {<10509.21.0>, [{registered_name,code_server}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7595254 (code_server:loop/1 + 64)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d3eb80 Return addr 0x081f9e74 ()", "y(0) {state,<0.11.0>,\"/opt/membase/1.7.0r/erlang/lib/erlang\",[\"/opt/membase/1.7.0r/bin", "y(1) <0.11.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,48}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[<10509.11.0>]}, {memory,131692}, {message_queue_len,0}, {reductions,206938}, {trap_exit,true}]}, {<10509.22.0>, [{registered_name,standard_error_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf2790 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,standard_error,<0.23.0>,<0.23.0>,{local,standard_error_sup}}", "y(4) standard_error_sup","y(5) <0.11.0>", "0xb7cf27ac Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.11.0>,<10509.23.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10509.23.0>, [{registered_name,standard_error}, {status,waiting}, {initial_call,{standard_error,server,2}}, {backtrace, ["Program counter: 0xb75925d4 (standard_error:server_loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1183c Return addr 0x081f9e74 ()", "y(0) #Port<0.506>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.22.0>,#Port<10509.506>]}, {memory,1352}, {message_queue_len,0}, {reductions,7}, {trap_exit,true}]}, {<10509.24.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d15bb0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,user_sup,<0.25.0>,<0.25.0>,{<0.24.0>,user_sup}}", "y(4) <0.24.0>","y(5) <0.11.0>", "0xb7d15bcc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<10509.11.0>,<10509.25.0>]}, {memory,6808}, {message_queue_len,0}, {reductions,71}, {trap_exit,true}]}, {<10509.25.0>, [{registered_name,user}, {status,waiting}, {initial_call,{user,server,2}}, {backtrace, ["Program counter: 0xb75a60cc (user:server_loop/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d3aa20 Return addr 0x081f9e74 ()", "y(0) []","y(1) []","y(2) {[],[]}", "y(3) #Port<0.523>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,6765}, {total_heap_size,13530}, {links,[<10509.6.0>,<10509.24.0>,#Port<10509.523>]}, {memory,54560}, {message_queue_len,0}, {reductions,28052}, {trap_exit,true}]}, {<10509.26.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce73a0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) kernel_config","y(3) []", "y(4) <0.26.0>","y(5) <0.11.0>", "0xb7ce73bc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.11.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,176}, {trap_exit,true}]}, {<10509.27.0>, [{registered_name,kernel_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d65c30 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,kernel_safe_sup},one_for_one,[{child,<0.92.0>,dets,{dets_server,sta", "y(4) kernel_safe_sup","y(5) <0.11.0>", "0xb7d65c4c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10509.87.0>,<10509.91.0>,<10509.92.0>,<10509.88.0>, <10509.50.0>,<10509.56.0>,<10509.11.0>]}, {memory,3536}, {message_queue_len,0}, {reductions,438}, {trap_exit,true}]}, {<10509.32.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1147c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.33.0>,{appl_data,sasl,[sasl_sup,alarm_handler,overload,release_handler]", "y(2) <0.7.0>", "0xb7d1148c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.7.0>,<10509.33.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<10509.33.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfa8f8 Return addr 0x081f9e74 ()", "y(0) {state,tty,undefined}","y(1) sasl", "y(2) <0.34.0>","y(3) <0.32.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.32.0>,<10509.34.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,70}, {trap_exit,true}]}, {<10509.34.0>, [{registered_name,sasl_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d108a8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_sup},one_for_one,[{child,<0.38.0>,release_handler,{release_han", "y(4) sasl_sup","y(5) <0.33.0>", "0xb7d108c4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.35.0>,<10509.38.0>,<10509.33.0>]}, {memory,2880}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<10509.35.0>, [{registered_name,sasl_safe_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10ff8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,sasl_safe_sup},one_for_one,[{child,<0.37.0>,overload,{overload,star", "y(4) sasl_safe_sup","y(5) <0.34.0>", "0xb7d11014 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.36.0>,<10509.37.0>,<10509.34.0>]}, {memory,2880}, {message_queue_len,0}, {reductions,174}, {trap_exit,true}]}, {<10509.36.0>, [{registered_name,alarm_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10c54 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,alarm_handler,false,[{system_memory_high_watermark,[]}],false}]", "y(3) alarm_handler","y(4) <0.35.0>", "0xb7d10c6c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.35.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,43}, {trap_exit,true}]}, {<10509.37.0>, [{registered_name,overload}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf2ee0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) overload", "y(3) {state,0,0,8.000000e-01,171,1.000000e-01,{0,0},clear}", "y(4) overload","y(5) <0.35.0>", "0xb7cf2efc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.35.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,39}, {trap_exit,false}]}, {<10509.38.0>, [{registered_name,release_handler}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1f758 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) release_handler", "y(3) {state,[],\"/opt/membase/1.7.0r/erlang/lib/erlang\",\"/opt/membase/1.7.0r/erlang/lib", "y(4) release_handler","y(5) <0.34.0>", "0xb7d1f774 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,4181}, {links,[<10509.34.0>]}, {memory,17124}, {message_queue_len,0}, {reductions,2238}, {trap_exit,false}]}, {<10509.41.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf23f4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.42.0>,{appl_data,os_mon,[os_mon_sup,os_mon_sysinfo,disksup,memsup,cpu_s", "y(2) <0.7.0>", "0xb7cf2404 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.7.0>,<10509.42.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,23}, {trap_exit,true}]}, {<10509.42.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfaca0 Return addr 0x081f9e74 ()", "y(0) []","y(1) os_mon","y(2) <0.43.0>", "y(3) <0.41.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.41.0>,<10509.43.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10509.43.0>, [{registered_name,os_mon_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d34058 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,os_mon_sup},one_for_one,[{child,<0.47.0>,cpu_sup,{cpu_sup,start_lin", "y(4) os_mon_sup","y(5) <0.42.0>", "0xb7d34074 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links, [<10509.44.0>,<10509.45.0>,<10509.47.0>,<10509.42.0>]}, {memory,3476}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10509.44.0>, [{registered_name,disksup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6db8d64 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) disksup", "y(3) {state,80,60000,{unix,linux},[{\"/\",11173104,41},{\"/boot\",101086,13},{\"/dev/shm\",1", "y(4) disksup","y(5) <0.43.0>", "0xb6db8d80 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,32}]}, {heap_size,2584}, {total_heap_size,5168}, {links,[<10509.43.0>,#Port<10509.876>]}, {memory,21092}, {message_queue_len,0}, {reductions,27119}, {trap_exit,true}]}, {<10509.45.0>, [{registered_name,memsup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb769597c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) memsup", "y(3) {state,{unix,linux},true,{1765871616,2075373568},{<0.414.0>,786412},false,60000,3", "y(4) memsup","y(5) <0.43.0>", "0xb7695998 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,36}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<10509.43.0>,<10509.46.0>]}, {memory,13196}, {message_queue_len,0}, {reductions,43610}, {trap_exit,true}]}, {<10509.46.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7611e94 (memsup:port_idle/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1d504 Return addr 0x081f9e74 ()", "y(0) []","y(1) #Port<0.905>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,34}]}, {heap_size,2584}, {total_heap_size,2961}, {links,[<10509.45.0>,#Port<10509.905>]}, {memory,12204}, {message_queue_len,0}, {reductions,12223}, {trap_exit,true}]}, {<10509.47.0>, [{registered_name,cpu_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d17850 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) cpu_sup", "y(3) {state,<0.48.0>,{unix,linux}}", "y(4) cpu_sup","y(5) <0.43.0>", "0xb7d1786c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.43.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,33}, {trap_exit,true}]}, {<10509.48.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb760a4f4 (cpu_sup:measurement_server_loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf362c Return addr 0x081f9e74 ()", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []","y(5) []", "y(6) []","y(7) []", "y(8) {internal,<0.49.0>,[],{unix,linux}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.49.0>]}, {memory,1272}, {message_queue_len,0}, {reductions,11}, {trap_exit,true}]}, {<10509.49.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb760ade0 (cpu_sup:port_server_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cea5b0 Return addr 0x081f9e74 ()", "y(0) []","y(1) 6000", "y(2) #Port<0.943>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10509.48.0>,#Port<10509.943>]}, {memory,4308}, {message_queue_len,0}, {reductions,299}, {trap_exit,false}]}, {<10509.50.0>, [{registered_name,timer_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf6ac4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) 476","y(2) timer", "y(3) []","y(4) timer_server", "y(5) <0.27.0>", "0xb7cf6ae0 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5033}]}, {heap_size,2584}, {total_heap_size,2961}, {links, [<10509.458.0>,<10509.478.0>,<10509.1161.0>, <10509.465.0>,<10509.415.0>,<10509.440.0>, <10509.457.0>,<10509.419.0>,<10509.78.0>, <10509.82.0>,<10509.27.0>]}, {memory,12444}, {message_queue_len,0}, {reductions,1631354}, {trap_exit,true}]}, {<10509.52.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d66f54 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.53.0>,{appl_data,ns_server,[ns_server_sup,ns_config,ns_config_sup,ns_co", "y(2) <0.7.0>", "0xb7d66f64 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.7.0>,<10509.53.0>]}, {memory,2860}, {message_queue_len,0}, {reductions,48}, {trap_exit,true}]}, {<10509.53.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cea958 Return addr 0x081f9e74 ()", "y(0) []","y(1) ns_server", "y(2) <0.54.0>","y(3) <0.52.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.52.0>,<10509.54.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,50}, {trap_exit,true}]}, {<10509.54.0>, [{registered_name,ns_server_cluster_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d31b9c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_cluster_sup},one_for_one,[{child,<0.414.0>,ns_server_sup,", "y(4) ns_server_cluster_sup","y(5) <0.53.0>", "0xb7d31bb8 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,2584}, {total_heap_size,2584}, {links, [<10509.65.0>,<10509.117.0>,<10509.414.0>, <10509.55.0>,<10509.64.0>,<10509.53.0>]}, {memory,10836}, {message_queue_len,0}, {reductions,1970}, {trap_exit,true}]}, {<10509.55.0>, [{registered_name,dist_manager}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d10500 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) dist_manager", "y(3) {state,true,\"10.1.5.178\"}", "y(4) dist_manager","y(5) <0.54.0>", "0xb7d1051c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.54.0>]}, {memory,2840}, {message_queue_len,0}, {reductions,912}, {trap_exit,false}]}, {<10509.56.0>, [{registered_name,inet_gethost_native_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf2b38 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_bridge", "y(3) {state,inet_gethost_native,<0.57.0>,<0.57.0>,{local,inet_gethost_native_sup}}", "y(4) inet_gethost_native_sup", "y(5) <0.27.0>", "0xb7cf2b54 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.27.0>,<10509.57.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,41}, {trap_exit,true}]}, {<10509.57.0>, [{registered_name,inet_gethost_native}, {status,waiting}, {initial_call,{inet_gethost_native,server_init,2}}, {backtrace, ["Program counter: 0xb7f44cc4 (inet_gethost_native:main_loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76caf84 Return addr 0x081f9e74 ()", "y(0) {state,#Port<0.984>,8000,8209,12306,<0.56.0>,4,{statistics,0,0,0,0,0,0,0,0}}"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,52}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10509.56.0>,#Port<10509.984>]}, {memory,8316}, {message_queue_len,0}, {reductions,11301}, {trap_exit,true}]}, {<10509.58.0>, [{registered_name,net_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce8900 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,net_sup},one_for_all,[{child,<0.61.0>,net_kernel,{net_kernel,start_", "y(4) net_sup","y(5) <0.11.0>", "0xb7ce891c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links, [<10509.59.0>,<10509.60.0>,<10509.61.0>,<10509.11.0>]}, {memory,4408}, {message_queue_len,0}, {reductions,252}, {trap_exit,true}]}, {<10509.59.0>, [{registered_name,erl_epmd}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce8ca8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) erl_epmd", "y(3) {state,#Port<0.1020>,21100,ns_1}", "y(4) erl_epmd","y(5) <0.58.0>", "0xb7ce8cc4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.58.0>,#Port<10509.1020>]}, {memory,1352}, {message_queue_len,0}, {reductions,127}, {trap_exit,false}]}, {<10509.60.0>, [{registered_name,auth}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d0d318 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) auth", "y(3) {state,lulxzmudqfmlvwvs,16403}", "y(4) auth","y(5) <0.58.0>", "0xb7d0d334 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.58.0>]}, {memory,2840}, {message_queue_len,0}, {reductions,379}, {trap_exit,true}]}, {<10509.61.0>, [{registered_name,net_kernel}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf1314 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) net_kernel", "y(3) {state,'ns_1@10.1.5.178','ns_1@10.1.5.178',longnames,{tick,<0.63.0>,15000},7000,s", "y(4) net_kernel","y(5) <0.58.0>", "0xb7cf1330 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10509.63.0>,<10509.411.0>,<10509.423.0>, <10509.58.0>,<10509.62.0>,#Port<10509.1018>]}, {memory,4464}, {message_queue_len,0}, {reductions,1999}, {trap_exit,true}]}, {<10509.62.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,accept_loop,2}}, {backtrace, ["Program counter: 0xb7e67f40 (prim_inet:accept0/2 + 92)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d588c8 Return addr 0xb762fcf4 (inet_tcp:accept/1 + 20)", "y(0) 84","y(1) #Port<0.1018>", "0xb7d588d4 Return addr 0xb7632ff0 (inet_tcp_dist:accept_loop/2 + 48)", "y(0) []", "0xb7d588dc Return addr 0x081f9e74 ()", "y(0) []","y(1) #Port<0.1018>", "y(2) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10509.61.0>]}, {memory,3396}, {message_queue_len,0}, {reductions,789}, {trap_exit,false}]}, {<10509.63.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{net_kernel,ticker,2}}, {backtrace, ["Program counter: 0xb75857dc (net_kernel:ticker_loop/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce9650 Return addr 0x081f9e74 ()", "y(0) 15000","y(1) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.61.0>]}, {memory,1272}, {message_queue_len,0}, {reductions,151}, {trap_exit,false}]}, {<10509.64.0>, [{registered_name,ns_cluster}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76d0178 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_cluster","y(3) {state}", "y(4) ns_cluster","y(5) <0.54.0>", "0xb76d0194 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,4181}, {total_heap_size,4181}, {links,[<10509.54.0>]}, {memory,17124}, {message_queue_len,0}, {reductions,1605}, {trap_exit,false}]}, {<10509.65.0>, [{registered_name,mb_mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d32530 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mb_mnesia_sup},one_for_one,[{child,<0.67.0>,mb_mnesia,{mb_mnesia,st", "y(4) mb_mnesia_sup","y(5) <0.54.0>", "0xb7d3254c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.66.0>,<10509.67.0>,<10509.54.0>]}, {memory,2880}, {message_queue_len,0}, {reductions,158}, {trap_exit,true}]}, {<10509.66.0>, [{registered_name,mb_mnesia_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d44384 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []","y(2) []", "y(3) mb_mnesia_events","y(4) <0.65.0>", "0xb7d4439c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.65.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,32}, {trap_exit,true}]}, {<10509.67.0>, [{registered_name,mb_mnesia}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76b9260 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mb_mnesia", "y(3) {state,['ns_1@10.1.5.178']}", "y(4) mb_mnesia","y(5) <0.65.0>", "0xb76b927c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,6}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10509.65.0>,<10509.76.0>]}, {memory,10756}, {message_queue_len,0}, {reductions,4729}, {trap_exit,true}]}, {<10509.70.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7efa3b0 (application_master:main_loop/2 + 32)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d45814 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []", "y(1) {state,<0.71.0>,{appl_data,mnesia,[mnesia_dumper_load_regulator,mnesia_event,mnes", "y(2) <0.7.0>", "0xb7d45824 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.7.0>,<10509.71.0>]}, {memory,2860}, {message_queue_len,0}, {reductions,45}, {trap_exit,true}]}, {<10509.71.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{application_master,start_it,4}}, {backtrace, ["Program counter: 0xb7efb6f4 (application_master:loop_it/4 + 40)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cead00 Return addr 0x081f9e74 ()", "y(0) {normal,[]}","y(1) mnesia_sup", "y(2) <0.72.0>","y(3) <0.70.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.70.0>,<10509.72.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,40}, {trap_exit,true}]}, {<10509.72.0>, [{registered_name,mnesia_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d646d8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_sup},one_for_all,[{child,<0.74.0>,mnesia_kernel_sup,{mnesia_", "y(4) mnesia_sup","y(5) <0.71.0>", "0xb7d646f4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10509.73.0>,<10509.74.0>,<10509.71.0>]}, {memory,3456}, {message_queue_len,0}, {reductions,207}, {trap_exit,true}]}, {<10509.73.0>, [{registered_name,mnesia_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cedd84 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,mnesia_event,false,{state,[],false,[]},false}]", "y(3) mnesia_event","y(4) <0.72.0>", "0xb7cedd9c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10509.72.0>,<10509.76.0>]}, {memory,3436}, {message_queue_len,0}, {reductions,408}, {trap_exit,true}]}, {<10509.74.0>, [{registered_name,mnesia_kernel_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d1311c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_kernel_sup},one_for_all,[{child,<0.83.0>,mnesia_late_loader,", "y(4) mnesia_kernel_sup","y(5) <0.72.0>", "0xb7d13138 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10509.77.0>,<10509.81.0>,<10509.82.0>,<10509.83.0>, <10509.79.0>,<10509.80.0>,<10509.78.0>,<10509.75.0>, <10509.76.0>,<10509.72.0>]}, {memory,4528}, {message_queue_len,0}, {reductions,599}, {trap_exit,true}]}, {<10509.75.0>, [{registered_name,mnesia_monitor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfec84 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_monitor", "y(3) {state,<0.74.0>,[],[],true,[],undefined,[]}", "y(4) mnesia_monitor","y(5) <0.74.0>", "0xb7cfeca0 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,28}]}, {heap_size,2584}, {total_heap_size,3194}, {links,[<10509.89.0>,<10509.74.0>]}, {memory,13196}, {message_queue_len,0}, {reductions,6495}, {trap_exit,true}]}, {<10509.76.0>, [{registered_name,mnesia_subscr}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d45460 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_subscr", "y(3) {state,<0.74.0>,20503}", "y(4) mnesia_subscr","y(5) <0.74.0>", "0xb7d4547c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.73.0>,<10509.74.0>,<10509.67.0>]}, {memory,1372}, {message_queue_len,0}, {reductions,111}, {trap_exit,true}]}, {<10509.77.0>, [{registered_name,mnesia_locker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb73bd100 (mnesia_locker:loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7678a28 Return addr 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(0) []","y(1) []","y(2) []", "y(3) []","y(4) []", "y(5) {state,<0.74.0>}", "0xb7678a44 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(1) mnesia_locker","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0xb7678a5c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,354}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10509.74.0>]}, {memory,8296}, {message_queue_len,0}, {reductions,193404}, {trap_exit,true}]}, {<10509.78.0>, [{registered_name,mnesia_recover}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d19390 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_recover", "y(3) {state,<0.74.0>,undefined,undefined,undefined,0,true,[]}", "y(4) mnesia_recover","y(5) <0.74.0>", "0xb7d193ac Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10509.74.0>,<10509.50.0>]}, {memory,5876}, {message_queue_len,0}, {reductions,2849}, {trap_exit,true}]}, {<10509.79.0>, [{registered_name,mnesia_tm}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb73e6d44 (mnesia_tm:doit_loop/1 + 108)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d4231c Return addr 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(0) []","y(1) []", "y(2) {state,{0,nil},{0,nil},<0.74.0>,[],[],[]}", "y(3) []","y(4) []","y(5) <0.74.0>", "y(6) {0,nil}","y(7) {0,nil}", "0xb7d42340 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(1) mnesia_tm","y(2) []","y(3) []", "y(4) [<0.74.0>]", "0xb7d42358 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,106}]}, {heap_size,2584}, {total_heap_size,2961}, {links,[<10509.74.0>]}, {memory,12244}, {message_queue_len,0}, {reductions,444677}, {trap_exit,true}]}, {<10509.80.0>, [{registered_name,mnesia_checkpoint_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d593b0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_checkpoint_sup},simple_one_for_one,[{child,undefined,mnesia_", "y(4) mnesia_checkpoint_sup","y(5) <0.74.0>", "0xb7d593cc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.74.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10509.81.0>, [{registered_name,mnesia_snmp_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf3288 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,mnesia_snmp_sup},simple_one_for_one,[{child,undefined,mnesia_snmp_s", "y(4) mnesia_snmp_sup","y(5) <0.74.0>", "0xb7cf32a4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.74.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,61}, {trap_exit,true}]}, {<10509.82.0>, [{registered_name,mnesia_controller}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6fda380 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mnesia_controller", "y(3) {state,<0.74.0>,true,[],[],{0,nil},[],[],{0,nil},undefined,[],[],{interval,#Ref<0", "y(4) mnesia_controller","y(5) <0.74.0>", "0xb6fda39c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,5}]}, {heap_size,377}, {total_heap_size,754}, {links,[<10509.74.0>,<10509.50.0>]}, {memory,3436}, {message_queue_len,0}, {reductions,943}, {trap_exit,true}]}, {<10509.83.0>, [{registered_name,mnesia_late_loader}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb738764c (mnesia_late_loader:loop/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cee704 Return addr 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(0) []","y(1) []", "y(2) {state,<0.74.0>}", "0xb7cee714 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7380c28 (mnesia_sp:init_proc/4 + 132)", "y(1) mnesia_late_loader","y(2) []", "y(3) []","y(4) [<0.74.0>]", "0xb7cee72c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,610}, {links,[<10509.74.0>]}, {memory,2840}, {message_queue_len,0}, {reductions,174}, {trap_exit,false}]}, {<10509.87.0>, [{registered_name,disk_log_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d5fd80 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,disk_log_sup},simple_one_for_one,[{child,undefined,disk_log,{disk_l", "y(4) disk_log_sup","y(5) <0.27.0>", "0xb7d5fd9c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,19}]}, {heap_size,987}, {total_heap_size,1597}, {links,[<10509.89.0>,<10509.27.0>]}, {memory,6808}, {message_queue_len,0}, {reductions,3694}, {trap_exit,true}]}, {<10509.88.0>, [{registered_name,disk_log_server}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb769ac98 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) disk_log_server","y(3) {state,[]}", "y(4) disk_log_server","y(5) <0.27.0>", "0xb769acb4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,45}]}, {heap_size,987}, {total_heap_size,1974}, {links,[<10509.89.0>,<10509.27.0>]}, {memory,8316}, {message_queue_len,0}, {reductions,3975}, {trap_exit,true}]}, {<10509.89.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb741f800 (disk_log:loop/1 + 84)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d05c40 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) {state,[],[],<0.87.0>,<0.88.0>,84,{arg,latest_log,undefined,\"/var/membase/mnesia/", "0xb7d05c48 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,610}, {total_heap_size,1597}, {links, [<10509.87.0>,<10509.88.0>,<10509.75.0>, #Port<10509.2892>]}, {memory,6848}, {message_queue_len,0}, {reductions,139297}, {trap_exit,true}]}, {<10509.91.0>, [{registered_name,dets_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76be338 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,dets_sup},simple_one_for_one,[{child,undefined,dets,{dets,istart_li", "y(4) dets_sup","y(5) <0.27.0>", "0xb76be354 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10509.27.0>]}, {memory,4348}, {message_queue_len,0}, {reductions,1139}, {trap_exit,true}]}, {<10509.92.0>, [{registered_name,dets}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76cbed8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) dets_server", "y(3) {state,40996,[<0.27.0>],[]}", "y(4) dets","y(5) <0.27.0>", "0xb76cbef4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,987}, {total_heap_size,1364}, {links,[<10509.27.0>]}, {memory,5856}, {message_queue_len,0}, {reductions,1613}, {trap_exit,true}]}, {<10509.117.0>, [{registered_name,ns_config_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d52c28 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_config_sup},rest_for_one,[{child,undefined,ns_config_log,{ns_con", "y(4) ns_config_sup","y(5) <0.54.0>", "0xb7d52c44 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,6765}, {total_heap_size,6765}, {links,[<10509.118.0>,<10509.119.0>,<10509.54.0>]}, {memory,27500}, {message_queue_len,0}, {reductions,580}, {trap_exit,true}]}, {<10509.118.0>, [{registered_name,ns_config_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0x081f9e70 (unknown function)", "CP: 0x081f9e74 ()", "arity = 3"," proc_lib"," wake_up", " [gen_event,wake_hib,[<0.117.0>,ns_config_events,[{handler,ns_pubsub,#Ref<0.0.0.6817>,{s"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,779}, {total_heap_size,779}, {links, [<10509.440.0>,<10509.475.0>,<10509.481.0>, <10509.117.0>]}, {memory,3656}, {message_queue_len,0}, {reductions,251580}, {trap_exit,true}]}, {<10509.119.0>, [{registered_name,ns_config}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6de7acc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_config", "y(3) {config,{full,\"/etc/membase/priv/config\",undefined,ns_config_default},[[{director", "y(4) ns_config","y(5) <0.117.0>", "0xb6de7ae8 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,11}]}, {heap_size,46368}, {total_heap_size,167761}, {links,[<10509.117.0>]}, {memory,671444}, {message_queue_len,0}, {reductions,84833}, {trap_exit,true}]}, {<10509.411.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_accept,6}}, {backtrace, ["Program counter: 0xb726b6e4 (dist_util:con_loop/9 + 72)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6f88f08 Return addr 0x081f9e74 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,9040,4348,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.178'", "y(6) {net_address,{{10,1,5,192},37359},\"10.1.5.192\",tcp,inet}", "y(7) #Port<0.2378>", "y(8) 'ns_1@10.1.5.192'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,7}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10509.61.0>,#Port<10509.2378>]}, {memory,8256}, {message_queue_len,0}, {reductions,1772}, {trap_exit,false}]}, {<10509.414.0>, [{registered_name,ns_server_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb70e3880 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_server_sup},one_for_one,[{child,<0.483.0>,moxi_stats_collector,{", "y(4) ns_server_sup","y(5) <0.54.0>", "0xb70e389c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,75025}, {total_heap_size,196418}, {links, [<10509.460.0>,<10509.474.0>,<10509.480.0>, <10509.481.0>,<10509.483.0>,<10509.477.0>, <10509.478.0>,<10509.475.0>,<10509.472.0>, <10509.473.0>,<10509.466.0>,<10509.417.0>, <10509.457.0>,<10509.458.0>,<10509.440.0>, <10509.415.0>,<10509.416.0>,<10509.54.0>]}, {memory,786412}, {message_queue_len,0}, {reductions,85274}, {trap_exit,true}]}, {<10509.415.0>, [{registered_name,ns_log}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76d42d0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity","y(2) ns_log", "y(3) {state,[{log_entry,{1302,303540,831821},'ns_1@10.1.5.178',ns_node_disco,3,\"Initia", "y(4) ns_log","y(5) <0.414.0>", "0xb76d42ec Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,10}]}, {heap_size,4181}, {total_heap_size,10946}, {links,[<10509.50.0>,<10509.414.0>]}, {memory,44204}, {message_queue_len,0}, {reductions,3317}, {trap_exit,false}]}, {<10509.416.0>, [{registered_name,ns_log_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d04cbc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []","y(2) []", "y(3) ns_log_events","y(4) <0.414.0>", "0xb7d04cd4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,2}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10509.414.0>]}, {memory,4348}, {message_queue_len,0}, {reductions,44}, {trap_exit,true}]}, {<10509.417.0>, [{registered_name,ns_node_disco_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d2ce78 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_node_disco_sup},rest_for_one,[{child,<0.430.0>,ns_config_rep,{ns", "y(4) ns_node_disco_sup","y(5) <0.414.0>", "0xb7d2ce94 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,233}, {total_heap_size,610}, {links, [<10509.419.0>,<10509.430.0>,<10509.418.0>, <10509.414.0>]}, {memory,2900}, {message_queue_len,0}, {reductions,793}, {trap_exit,true}]}, {<10509.418.0>, [{registered_name,ns_node_disco_events}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d2eb1c Return addr 0xb7e88088 (proc_lib:wake_up/3 + 60)", "y(0) false","y(1) []", "y(2) [{handler,menelaus_event,ns_node_disco_events,{state,ns_node_disco_events,undefin", "y(3) ns_node_disco_events","y(4) <0.417.0>", "0xb7d2eb34 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88098 (proc_lib:wake_up/3 + 76)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,233}, {total_heap_size,322}, {links,[<10509.417.0>]}, {memory,1768}, {message_queue_len,0}, {reductions,73}, {trap_exit,true}]}, {<10509.419.0>, [{registered_name,ns_node_disco}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7164c70 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_node_disco", "y(3) {state,false,{interval,#Ref<0.0.0.6490>}}", "y(4) ns_node_disco","y(5) <0.417.0>", "0xb7164c8c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,45}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10509.417.0>,<10509.50.0>]}, {memory,300520}, {message_queue_len,0}, {reductions,3737}, {trap_exit,false}]}, {<10509.423.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{inet_tcp_dist,do_accept,6}}, {backtrace, ["Program counter: 0xb726b6e4 (dist_util:con_loop/9 + 72)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d2e768 Return addr 0x081f9e74 ()", "y(0) []", "y(1) #Fun", "y(2) #Fun", "y(3) {tick,2001,2016,2,2}","y(4) normal", "y(5) 'ns_1@10.1.5.178'", "y(6) {net_address,{{10,1,5,193},38404},\"10.1.5.193\",tcp,inet}", "y(7) #Port<0.2383>", "y(8) 'ns_1@10.1.5.193'","y(9) <0.61.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10509.61.0>,#Port<10509.2383>]}, {memory,8256}, {message_queue_len,0}, {reductions,1772}, {trap_exit,false}]}, {<10509.430.0>, [{registered_name,ns_config_rep}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6e423dc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_config_rep","y(3) {state}", "y(4) ns_config_rep","y(5) <0.417.0>", "0xb6e423f8 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,14}]}, {heap_size,46368}, {total_heap_size,92736}, {links,[<10509.417.0>]}, {memory,371344}, {message_queue_len,0}, {reductions,15836}, {trap_exit,false}]}, {<10509.440.0>, [{registered_name,mb_master}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb71d63b4 (gen_fsm:loop/7 + 156)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6ded51c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mb_master", "y(3) {state,undefined,'ns_1@10.1.5.192',['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.", "y(4) candidate","y(5) mb_master", "y(6) <0.414.0>", "0xb6ded53c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,49}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10509.118.0>,<10509.414.0>,<10509.50.0>]}, {memory,8336}, {message_queue_len,0}, {reductions,40274}, {trap_exit,false}]}, {<10509.457.0>, [{registered_name,ns_heart}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6dafb40 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_heart", "y(3) [{meminfo,<<771 bytes>>},{replication,[{\"FailoverNodesTests-4a448bb0-d65f-443f-9c", "y(4) ns_heart","y(5) <0.414.0>", "0xb6dafb5c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,6765}, {total_heap_size,35422}, {links,[<10509.414.0>,<10509.50.0>]}, {memory,142108}, {message_queue_len,0}, {reductions,1412474}, {trap_exit,false}]}, {<10509.458.0>, [{registered_name,ns_doctor}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb716b628 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_doctor", "y(3) {state,{dict,3,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{", "y(4) ns_doctor","y(5) <0.414.0>", "0xb716b644 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,6765}, {total_heap_size,17711}, {links,[<10509.414.0>,<10509.50.0>]}, {memory,71264}, {message_queue_len,0}, {reductions,56402}, {trap_exit,false}]}, {<10509.460.0>, [{registered_name,menelaus_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6debc28 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,menelaus_sup},one_for_one,[{child,<0.465.0>,menelaus_web_alerts_srv", "y(4) menelaus_sup","y(5) <0.414.0>", "0xb6debc44 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,8}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [<10509.461.0>,<10509.465.0>,<10509.1161.0>, <10509.414.0>]}, {memory,131812}, {message_queue_len,0}, {reductions,10851}, {trap_exit,true}]}, {<10509.461.0>, [{registered_name,menelaus_web}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf9f4c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) mochiweb_socket_server", "y(3) {mochiweb_socket_server,8091,#Fun,{local,menelaus_web}", "y(4) menelaus_web","y(5) <0.460.0>", "0xb7cf9f68 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,987}, {links, [<10509.460.0>,<10509.479.0>,<10509.485.0>, <10509.462.0>,#Port<10509.2388>]}, {memory,4428}, {message_queue_len,0}, {reductions,349}, {trap_exit,true}]}, {<10509.462.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb720b578 (menelaus_web:handle_streaming/4 + 196)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6e0a424 Return addr 0xb7209848 (menelaus_web:loop/3 + 13188)", "y(0) {struct,[{buckets,[]}]}", "y(1) {struct,[{buckets,[]}]}", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBuck", "y(3) {mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(4) #Fun", "0xb6e0a43c Return addr 0xb71f5880 (mochiweb_http:headers/5 + 680)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "y(5) Catch 0xb7209858 (menelaus_web:loop/3 + 13204)", "0xb6e0a458 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2393>,'GET',\"/pools/default/saslBucketsStreaming\",{1,1}", "0xb6e0a46c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,28657}, {total_heap_size,28657}, {links,[<10509.461.0>,#Port<10509.2393>]}, {memory,115128}, {message_queue_len,0}, {reductions,5135}, {trap_exit,false}]}, {<10509.465.0>, [{registered_name,menelaus_web_alerts_srv}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7768100 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) menelaus_web_alerts_srv", "y(3) {state,[],[],{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],", "y(4) menelaus_web_alerts_srv", "y(5) <0.460.0>", "0xb776811c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,321}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<10509.50.0>,<10509.460.0>]}, {memory,33868}, {message_queue_len,0}, {reductions,187219}, {trap_exit,false}]}, {<10509.466.0>, [{registered_name,ns_port_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6ff66d8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_port_sup},one_for_one,[{child,<0.488.0>,{memcached,\"./bin/memcac", "y(4) ns_port_sup","y(5) <0.414.0>", "0xb6ff66f4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,28657}, {total_heap_size,75025}, {links,[<10509.468.0>,<10509.488.0>,<10509.414.0>]}, {memory,300540}, {message_queue_len,0}, {reductions,3234}, {trap_exit,true}]}, {<10509.468.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d5ee10 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi,5000,{1302,303852,128447},<0.469.0>}", "y(4) <0.468.0>","y(5) <0.466.0>", "0xb7d5ee2c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,987}, {total_heap_size,2584}, {links,[<10509.466.0>,<10509.469.0>]}, {memory,10756}, {message_queue_len,0}, {reductions,150}, {trap_exit,true}]}, {<10509.469.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cf95c0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2389>,moxi,{[\"2011-04-08 16:04:12: (cproxy_config.c.325) env: MOXI", "y(4) <0.469.0>","y(5) <0.468.0>", "0xb7cf95dc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10509.468.0>,#Port<10509.2389>]}, {memory,13196}, {message_queue_len,0}, {reductions,273}, {trap_exit,true}]}, {<10509.472.0>, [{registered_name,ns_tick_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6dedeb0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.6760>,{state,#Fun,ignored},<", "y(3) ns_tick_event","y(4) <0.414.0>", "0xb6dedec8 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,60}]}, {heap_size,610}, {total_heap_size,987}, {links,[<10509.477.0>,<10509.414.0>]}, {memory,4368}, {message_queue_len,0}, {reductions,10184}, {trap_exit,true}]}, {<10509.473.0>, [{registered_name,ns_stats_event}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8e794 (gen_event:fetch_msg/5 + 44)", "CP: 0x00000000 (invalid)","arity = 0", "0xb76908bc Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) false","y(1) []", "y(2) [{handler,ns_pubsub,#Ref<0.0.0.6806>,{state,#Fun,ignored},<", "y(3) ns_stats_event","y(4) <0.414.0>", "0xb76908d4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,420}]}, {heap_size,1597}, {total_heap_size,3194}, {links,[<10509.478.0>,<10509.414.0>]}, {memory,13196}, {message_queue_len,0}, {reductions,28769}, {trap_exit,true}]}, {<10509.474.0>, [{registered_name,ns_bucket_worker}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d43c4c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) work_queue","y(3) []", "y(4) ns_bucket_worker","y(5) <0.414.0>", "0xb7d43c68 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,610}, {total_heap_size,1597}, {links,[<10509.414.0>]}, {memory,6788}, {message_queue_len,0}, {reductions,288}, {trap_exit,false}]}, {<10509.475.0>, [{registered_name,ns_bucket_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb70f4d48 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_bucket_sup},one_for_one,[{child,<0.765.0>,{per_bucket_sup,\"Failo", "y(4) ns_bucket_sup","y(5) <0.414.0>", "0xb70f4d64 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<10509.414.0>,<10509.765.0>,<10509.118.0>]}, {memory,71284}, {message_queue_len,0}, {reductions,246}, {trap_exit,true}]}, {<10509.477.0>, [{registered_name,system_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d09798 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) system_stats_collector", "y(3) {state,#Port<0.2392>,[{cpu_local_ms,2425400020},{cpu_idle_ms,2395481690},{swap_to", "y(4) system_stats_collector", "y(5) <0.414.0>", "0xb7d097b4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,165}]}, {heap_size,1597}, {total_heap_size,1974}, {links,[<10509.414.0>,<10509.472.0>,#Port<10509.2392>]}, {memory,8336}, {message_queue_len,0}, {reductions,45922}, {trap_exit,false}]}, {<10509.478.0>, [{registered_name,'stats_archiver-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb6db3c98 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) stats_archiver", "y(3) {state,\"@system\"}", "y(4) 'stats_archiver-@system'", "y(5) <0.414.0>", "0xb6db3cb4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1883}]}, {heap_size,4181}, {total_heap_size,8362}, {links,[<10509.414.0>,<10509.473.0>,<10509.50.0>]}, {memory,33888}, {message_queue_len,0}, {reductions,4213863}, {trap_exit,false}]}, {<10509.479.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb720b578 (menelaus_web:handle_streaming/4 + 196)", "CP: 0x00000000 (invalid)","arity = 0", "0xb707e42c Return addr 0xb7240ed0 (menelaus_web_buckets:checking_bucket_access/4 + 260)", "y(0) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(1) {struct,[{name,<<55 bytes>>},{bucketType,membase},{authType,none},{saslPassword,<", "y(2) {mochiweb_response,{mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsS", "y(3) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) #Fun", "0xb707e444 Return addr 0xb72097a0 (menelaus_web:loop/3 + 13020)", "y(0) #Ref<0.0.0.6889>", "y(1) Catch 0xb7240f00 (menelaus_web_buckets:checking_bucket_access/4 + 308)", "y(2) []", "y(3) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(4) []", "0xb707e45c Return addr 0xb71f5880 (mochiweb_http:headers/5 + 680)", "y(0) []","y(1) []","y(2) []", "y(3) []", "y(4) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "y(5) Catch 0xb7209858 (menelaus_web:loop/3 + 13204)", "0xb707e478 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) #Fun", "y(1) []","y(2) []", "y(3) {mochiweb_request,#Port<0.2396>,'GET',\"/pools/default/bucketsStreaming/FailoverNo", "0xb707e48c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,3}]}, {heap_size,17711}, {total_heap_size,92736}, {links,[<10509.461.0>,#Port<10509.2396>]}, {memory,371444}, {message_queue_len,0}, {reductions,1220951}, {trap_exit,false}]}, {<10509.480.0>, [{registered_name,'stats_reader-@system'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7cfb620 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) stats_reader", "y(3) {state,\"@system\"}", "y(4) 'stats_reader-@system'", "y(5) <0.414.0>", "0xb7cfb63c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.414.0>]}, {memory,1332}, {message_queue_len,0}, {reductions,26}, {trap_exit,false}]}, {<10509.481.0>, [{registered_name,ns_moxi_sup}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7106208 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor", "y(3) {state,{local,ns_moxi_sup},one_for_one,[{child,<0.482.0>,{\"FailoverNodesTests-4a4", "y(4) ns_moxi_sup","y(5) <0.414.0>", "0xb7106224 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,17711}, {total_heap_size,17711}, {links,[<10509.414.0>,<10509.482.0>,<10509.118.0>]}, {memory,71284}, {message_queue_len,0}, {reductions,2236}, {trap_exit,true}]}, {<10509.482.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d046d0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2394>,moxi,{[\"2011-04-08 16:04:12: (cproxy_config.c.316) env: MOXI", "y(4) <0.482.0>","y(5) <0.481.0>", "0xb7d046ec Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,1597}, {total_heap_size,1597}, {links,[<10509.481.0>,#Port<10509.2394>]}, {memory,6808}, {message_queue_len,0}, {reductions,168}, {trap_exit,true}]}, {<10509.483.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7ce99e0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,moxi_stats_collector,5000,{1302,303852,147786},<0.484.0>}", "y(4) <0.483.0>","y(5) <0.414.0>", "0xb7ce99fc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.414.0>,<10509.484.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,57}, {trap_exit,true}]}, {<10509.484.0>, [{registered_name,moxi_stats_collector}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb775fe30 Return addr 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(0) []","y(1) infinity", "y(2) moxi_stats_collector", "y(3) {state,#Port<0.2395>,{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[", "y(4) <0.484.0>","y(5) <0.483.0>", "0xb775fe4c Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(1) moxi_stats_collector","y(2) []", "y(3) moxi_stats_collector", "y(4) {local,moxi_stats_collector}", "y(5) <0.483.0>","y(6) <0.483.0>", "0xb775fe6c Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,32}]}, {heap_size,4181}, {total_heap_size,32838}, {links, [#Port<10509.2438>,<10509.483.0>,#Port<10509.2395>]}, {memory,131792}, {message_queue_len,0}, {reductions,2259506}, {trap_exit,false}]}, {<10509.485.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e67f40 (prim_inet:accept0/2 + 92)", "CP: 0x00000000 (invalid)","arity = 0", "0xb769b780 Return addr 0xb762fcf4 (inet_tcp:accept/1 + 20)", "y(0) 97","y(1) #Port<0.2388>", "0xb769b78c Return addr 0xb71fcb04 (mochiweb_socket_server:acceptor_loop/1 + 80)", "y(0) []", "0xb769b794 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) #Fun", "y(1) <0.461.0>", "y(2) Catch 0xb71fcb04 (mochiweb_socket_server:acceptor_loop/1 + 80)", "0xb769b7a4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.461.0>]}, {memory,1372}, {message_queue_len,0}, {reductions,38}, {trap_exit,false}]}, {<10509.488.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d06ba0 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) supervisor_cushion", "y(3) {state,memcached,5000,{1302,303859,675513},<0.489.0>}", "y(4) <0.488.0>","y(5) <0.466.0>", "0xb7d06bbc Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10509.466.0>,<10509.489.0>]}, {memory,4368}, {message_queue_len,0}, {reductions,55}, {trap_exit,true}]}, {<10509.489.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7698db8 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) ns_port_server", "y(3) {state,#Port<0.2401>,memcached,{[empty],[empty,empty]},undefined,[],0}", "y(4) <0.489.0>","y(5) <0.488.0>", "0xb7698dd4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,987}, {total_heap_size,987}, {links,[<10509.488.0>,#Port<10509.2401>]}, {memory,4368}, {message_queue_len,0}, {reductions,58}, {trap_exit,true}]}, {<10509.765.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7e8844c (proc_lib:sync_wait/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d09b4c Return addr 0xb726f6b8 (single_bucket_sup:'-start_link/1-fun-0-'/2 + 172)", "y(0) infinity","y(1) <0.766.0>", "0xb7d09b58 Return addr 0x081f9e74 ()", "y(0) <0.475.0>", "y(1) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\""]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.475.0>,<10509.766.0>]}, {memory,1292}, {message_queue_len,0}, {reductions,30}, {trap_exit,true}]}, {<10509.766.0>, [{registered_name, 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7e8844c (proc_lib:sync_wait/2 + 28)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d65268 Return addr 0xb7f0395c (supervisor:do_start_child/2 + 156)", "y(0) infinity","y(1) <0.767.0>", "0xb7d65274 Return addr 0xb7f0367c (supervisor:start_children/3 + 76)", "y(0) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(2) Catch 0xb7f0395c (supervisor:do_start_child/2 + 156)", "0xb7d65284 Return addr 0xb7f03370 (supervisor:init_children/2 + 136)", "y(0) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(1) []", "y(2) [{child,undefined,{ns_vbm_sup,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b9582", "y(3) {child,undefined,{ns_memcached,\"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958", "0xb7d65298 Return addr 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(0) {state,{local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b95", "y(1) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "0xb7d652a4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(1) 'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'", "y(2) []","y(3) supervisor", "y(4) {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "y(5) <0.765.0>","y(6) <0.765.0>", "0xb7d652c4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,377}, {total_heap_size,987}, {links,[<10509.765.0>,<10509.767.0>]}, {memory,4368}, {message_queue_len,0}, {reductions,664}, {trap_exit,true}]}, {<10509.767.0>, [{registered_name,[]}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb761487c (timer:sleep/1 + 20)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7763f78 Return addr 0xb71ecc34 (ns_memcached:ensure_bucket/2 + 456)", "y(0) 1000", "0xb7763f80 Return addr 0xb71eabc0 (ns_memcached:init/1 + 72)", "y(0) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "y(1) #Port<0.2571>","y(2) []", "0xb7763f90 Return addr 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(0) []","y(1) {1302,304045,481732}", "y(2) #Port<0.2571>", "y(3) \"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561\"", "0xb7763fa4 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) Catch 0xb7ebf99c (gen_server:init_it/6 + 196)", "y(1) <0.767.0>","y(2) []", "y(3) ns_memcached","y(4) <0.767.0>", "y(5) <0.766.0>","y(6) <0.766.0>", "0xb7763fc4 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,1}]}, {heap_size,4181}, {total_heap_size,32838}, {links,[<10509.766.0>,#Port<10509.2571>]}, {memory,131772}, {message_queue_len,0}, {reductions,1428531}, {trap_exit,false}]}, {<10509.1161.0>, [{registered_name,hot_keys_keeper}, {status,waiting}, {initial_call,{proc_lib,init_p,5}}, {backtrace, ["Program counter: 0xb7ebfde4 (gen_server:loop/6 + 144)", "CP: 0x00000000 (invalid)","arity = 0", "0xb7d44728 Return addr 0xb7e88018 (proc_lib:init_p_do_apply/3 + 28)", "y(0) []","y(1) infinity", "y(2) hot_keys_keeper", "y(3) {state,[],undefined,undefined}", "y(4) hot_keys_keeper","y(5) <0.460.0>", "0xb7d44744 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7e88028 (proc_lib:init_p_do_apply/3 + 44)"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,0}]}, {heap_size,233}, {total_heap_size,233}, {links,[<10509.460.0>,<10509.50.0>]}, {memory,1352}, {message_queue_len,0}, {reductions,56}, {trap_exit,false}]}, {<10509.1162.0>, [{registered_name,[]}, {status,running}, {initial_call,{erlang,apply,2}}, {backtrace, ["Program counter: 0xb7df922c (unknown function)", "CP: 0xb71c1234 (diag_handler:grab_process_info/1 + 32)", "0xb65cc774 Return addr 0xb71c2508 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 44)", "y(0) []", "0xb65cc77c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) []","y(1) <0.1162.0>", "0xb65cc788 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,hot_keys_keeper},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.1161.0>", "0xb65cc794 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.767.0>", "0xb65cc7a0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-5", "y(1) <0.766.0>", "0xb65cc7ac Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.765.0>", "0xb65cc7b8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.489.0>", "0xb65cc7c4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.488.0>", "0xb65cc7d0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.485.0>", "0xb65cc7dc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,moxi_stats_collector},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.484.0>", "0xb65cc7e8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.483.0>", "0xb65cc7f4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.482.0>", "0xb65cc800 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_moxi_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.481.0>", "0xb65cc80c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,'stats_reader-@system'},{status,waiting},{initial_call,{proc_li", "y(1) <0.480.0>", "0xb65cc818 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.479.0>", "0xb65cc824 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,'stats_archiver-@system'},{status,waiting},{initial_call,{proc_", "y(1) <0.478.0>", "0xb65cc830 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,system_stats_collector},{status,waiting},{initial_call,{proc_li", "y(1) <0.477.0>", "0xb65cc83c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_bucket_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.475.0>", "0xb65cc848 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_bucket_worker},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.474.0>", "0xb65cc854 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_stats_event},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.473.0>", "0xb65cc860 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_tick_event},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.472.0>", "0xb65cc86c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.469.0>", "0xb65cc878 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.468.0>", "0xb65cc884 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_port_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}", "y(1) <0.466.0>", "0xb65cc890 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,menelaus_web_alerts_srv},{status,waiting},{initial_call,{proc_l", "y(1) <0.465.0>", "0xb65cc89c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.462.0>", "0xb65cc8a8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,menelaus_web},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.461.0>", "0xb65cc8b4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,menelaus_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.460.0>", "0xb65cc8c0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_doctor},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.458.0>", "0xb65cc8cc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_heart},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.457.0>", "0xb65cc8d8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_master},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.440.0>", "0xb65cc8e4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config_rep},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.430.0>", "0xb65cc8f0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_accept,6}}", "y(1) <0.423.0>", "0xb65cc8fc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_node_disco},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.419.0>", "0xb65cc908 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_node_disco_events},{status,waiting},{initial_call,{proc_lib,", "y(1) <0.418.0>", "0xb65cc914 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_node_disco_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.417.0>", "0xb65cc920 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_log_events},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.416.0>", "0xb65cc92c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_log},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.415.0>", "0xb65cc938 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_server_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.414.0>", "0xb65cc944 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,do_accept,6}}", "y(1) <0.411.0>", "0xb65cc950 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.119.0>", "0xb65cc95c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.118.0>", "0xb65cc968 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_config_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.117.0>", "0xb65cc974 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,dets},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.92.0>", "0xb65cc980 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,dets_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.91.0>", "0xb65cc98c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.89.0>", "0xb65cc998 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,disk_log_server},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.88.0>", "0xb65cc9a4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,disk_log_sup},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.87.0>", "0xb65cc9b0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_late_loader},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.83.0>", "0xb65cc9bc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_controller},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.82.0>", "0xb65cc9c8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_snmp_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.81.0>", "0xb65cc9d4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_checkpoint_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.80.0>", "0xb65cc9e0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_tm},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.79.0>", "0xb65cc9ec Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_recover},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.78.0>", "0xb65cc9f8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_locker},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.77.0>", "0xb65cca04 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_subscr},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.76.0>", "0xb65cca10 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_monitor},{status,waiting},{initial_call,{proc_lib,init_p", "y(1) <0.75.0>", "0xb65cca1c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_kernel_sup},{status,waiting},{initial_call,{proc_lib,ini", "y(1) <0.74.0>", "0xb65cca28 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_event},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.73.0>", "0xb65cca34 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.72.0>", "0xb65cca40 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.71.0>", "0xb65cca4c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.70.0>", "0xb65cca58 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_mnesia},{status,waiting},{initial_call,{proc_lib,init_p,5}},", "y(1) <0.67.0>", "0xb65cca64 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_mnesia_events},{status,waiting},{initial_call,{proc_lib,init", "y(1) <0.66.0>", "0xb65cca70 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,mb_mnesia_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.65.0>", "0xb65cca7c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_cluster},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.64.0>", "0xb65cca88 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{net_kernel,ticker,2}},{back", "y(1) <0.63.0>", "0xb65cca94 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{inet_tcp_dist,accept_loop,2", "y(1) <0.62.0>", "0xb65ccaa0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,net_kernel},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.61.0>", "0xb65ccaac Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,auth},{status,waiting},{initial_call,{proc_lib,init_p,5}},{back", "y(1) <0.60.0>", "0xb65ccab8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,erl_epmd},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.59.0>", "0xb65ccac4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,net_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.58.0>", "0xb65ccad0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,inet_gethost_native},{status,waiting},{initial_call,{inet_getho", "y(1) <0.57.0>", "0xb65ccadc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,inet_gethost_native_sup},{status,waiting},{initial_call,{proc_l", "y(1) <0.56.0>", "0xb65ccae8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,dist_manager},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.55.0>", "0xb65ccaf4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,ns_server_cluster_sup},{status,waiting},{initial_call,{proc_lib", "y(1) <0.54.0>", "0xb65ccb00 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.53.0>", "0xb65ccb0c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.52.0>", "0xb65ccb18 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,timer_server},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.50.0>", "0xb65ccb24 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.49.0>", "0xb65ccb30 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.48.0>", "0xb65ccb3c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,cpu_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.47.0>", "0xb65ccb48 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.46.0>", "0xb65ccb54 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,memsup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{ba", "y(1) <0.45.0>", "0xb65ccb60 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,disksup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.44.0>", "0xb65ccb6c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,os_mon_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.43.0>", "0xb65ccb78 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.42.0>", "0xb65ccb84 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.41.0>", "0xb65ccb90 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,release_handler},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.38.0>", "0xb65ccb9c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,overload},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.37.0>", "0xb65ccba8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,alarm_handler},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.36.0>", "0xb65ccbb4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,sasl_safe_sup},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.35.0>", "0xb65ccbc0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,sasl_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}},{", "y(1) <0.34.0>", "0xb65ccbcc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.33.0>", "0xb65ccbd8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.32.0>", "0xb65ccbe4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,kernel_safe_sup},{status,waiting},{initial_call,{proc_lib,init_", "y(1) <0.27.0>", "0xb65ccbf0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.26.0>", "0xb65ccbfc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,user},{status,waiting},{initial_call,{user,server,2}},{backtrac", "y(1) <0.25.0>", "0xb65ccc08 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.24.0>", "0xb65ccc14 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,standard_error},{status,waiting},{initial_call,{standard_error,", "y(1) <0.23.0>", "0xb65ccc20 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,standard_error_sup},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.22.0>", "0xb65ccc2c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,code_server},{status,waiting},{initial_call,{erlang,apply,2}},{", "y(1) <0.21.0>", "0xb65ccc38 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,file_server_2},{status,waiting},{initial_call,{proc_lib,init_p,", "y(1) <0.20.0>", "0xb65ccc44 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,global_group},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.19.0>", "0xb65ccc50 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,inet_db},{status,waiting},{initial_call,{proc_lib,init_p,5}},{b", "y(1) <0.17.0>", "0xb65ccc5c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.16.0>", "0xb65ccc68 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.15.0>", "0xb65ccc74 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{erlang,apply,2}},{backtrace", "y(1) <0.14.0>", "0xb65ccc80 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,global_name_server},{status,waiting},{initial_call,{proc_lib,in", "y(1) <0.13.0>", "0xb65ccc8c Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,rex},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backt", "y(1) <0.12.0>", "0xb65ccc98 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,kernel_sup},{status,waiting},{initial_call,{proc_lib,init_p,5}}", "y(1) <0.11.0>", "0xb65ccca4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{application_master,start_it", "y(1) <0.10.0>", "0xb65cccb0 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,[]},{status,waiting},{initial_call,{proc_lib,init_p,5}},{backtr", "y(1) <0.9.0>", "0xb65cccbc Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,application_controller},{status,waiting},{initial_call,{erlang,", "y(1) <0.7.0>", "0xb65cccc8 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,error_logger},{status,waiting},{initial_call,{proc_lib,init_p,5", "y(1) <0.6.0>", "0xb65cccd4 Return addr 0xb71c2528 (diag_handler:'-do_diag_per_node/0-lc$^0/1-0-'/1 + 76)", "y(0) [{registered_name,erl_prim_loader},{status,waiting},{initial_call,{erlang,apply,2", "y(1) <0.3.0>", "0xb65ccce0 Return addr 0xb71c133c (diag_handler:do_diag_per_node/0 + 112)", "y(0) [{registered_name,init},{status,waiting},{initial_call,{otp_ring0,start,2}},{back", "y(1) <0.0.0>", "0xb65cccec Return addr 0xb7f104f4 (rpc:'-handle_call_call/6-fun-0-'/5 + 104)", "y(0) []","y(1) []", "y(2) [{version,[{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"}", "y(3) [{alerts,[{email,[]},{email_alerts,false},{email_server,[{user,undefined},{pass,'", "y(4) [\"bucket_engine_1.6.5_10_g3c82cbf-Linux.i686.tar.gz\",\"curl-7.21.1-w64_patched.tar", "y(5) [{kernel,\"2.13.4\"},{mnesia,\"4.4.12\"},{stdlib,\"1.16.4\"},{os_mon,\"2.2.4\"},{ns_serve", "0xb65ccd08 Return addr 0x081f9e74 ()", "y(0) Catch 0xb7f104f4 (rpc:'-handle_call_call/6-fun-0-'/5 + 104)", "y(1) []","y(2) []","y(3) []", "y(4) <0.12.0>"]}, {error_handler,error_handler}, {garbage_collection, [{fullsweep_after,65535},{minor_gcs,4}]}, {heap_size,196418}, {total_heap_size,710647}, {links,[]}, {memory,2842948}, {message_queue_len,0}, {reductions,532690}, {trap_exit,false}]}]}, {memory,{2075373568,1765871616,{<10509.414.0>,786412}}}, {disk, [{"/",11173104,41}, {"/boot",101086,13}, {"/dev/shm",1013364,0}]}]}] nodes_info = [{struct,[{uptime,<<"1113">>}, {memoryTotal,2075373568}, {memoryFree,309501952}, {mcdMemoryReserved,1583}, {mcdMemoryAllocated,1583}, {otpNode,<<"ns_1@10.1.5.178">>}, {otpCookie,<<"lulxzmudqfmlvwvs">>}, {clusterMembership,<<"inactiveFailed">>}, {status,<<"healthy">>}, {hostname,<<"10.1.5.178:8091">>}, {clusterCompatibility,1}, {version,<<"1.6.5.3_291_gba5fe27">>}, {os,<<"i686-pc-linux-gnu">>}, {ports,{struct,[{proxy,11211},{direct,11210}]}}]}, {struct,[{uptime,<<"1192">>}, {memoryTotal,2058928128}, {memoryFree,464596992}, {mcdMemoryReserved,1570}, {mcdMemoryAllocated,1570}, {otpNode,<<"ns_1@10.1.5.192">>}, {otpCookie,<<"lulxzmudqfmlvwvs">>}, {clusterMembership,<<"active">>}, {status,<<"healthy">>}, {hostname,<<"10.1.5.192:8091">>}, {clusterCompatibility,1}, {version,<<"1.6.5.3_291_gba5fe27">>}, {os,<<"x86_64-unknown-linux-gnu">>}, {ports,{struct,[{proxy,11211},{direct,11210}]}}]}, {struct,[{uptime,<<"1160">>}, {memoryTotal,2058928128}, {memoryFree,989904896}, {mcdMemoryReserved,1570}, {mcdMemoryAllocated,1570}, {otpNode,<<"ns_1@10.1.5.193">>}, {otpCookie,<<"lulxzmudqfmlvwvs">>}, {clusterMembership,<<"inactiveFailed">>}, {status,<<"healthy">>}, {hostname,<<"10.1.5.193:8091">>}, {clusterCompatibility,1}, {version,<<"1.6.5.3_291_gba5fe27">>}, {os,<<"x86_64-unknown-linux-gnu">>}, {ports,{struct,[{proxy,11211},{direct,11210}]}}]}] buckets = [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], [undefined,undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined]]}, {fastForwardMap,undefined}]}] logs: ------------------------------- 2011-04-08 15:58:04.678 ns_node_disco:3:info:cookie update - Initial otp cookie generated: iafrpkdqnfrcgybi 2011-04-08 15:58:04.809 menelaus_app:1:info:web start ok - Membase Server has started on web port 8091 on node 'ns_1@10.1.5.193'. 2011-04-08 16:03:48.119 menelaus_app:1:info:web start ok - Membase Server has started on web port 8091 on node 'ns_1@10.1.5.193'. 2011-04-08 16:03:48.293 ns_cluster:3:info:message - Node ns_1@10.1.5.193 joined cluster 2011-04-08 16:03:48.492 supervisor_cushion:1:warning:port exited too soon after restart - Service moxi_stats_collector exited on node 'ns_1@10.1.5.193' in 0.20s 2011-04-08 16:03:48.492 supervisor_cushion:1:warning:port exited too soon after restart - Service moxi_stats_collector exited on node 'ns_1@10.1.5.193' in 0.20s 2011-04-08 16:03:49.836 ns_port_server:0:info:message - Port server memcached on node 'ns_1@10.1.5.193' exited with status 137. Restarting. Messages: 2011-04-08 16:03:49.836 supervisor_cushion:1:warning:port exited too soon after restart - Service memcached exited on node 'ns_1@10.1.5.193' in 1.67s 2011-04-08 16:03:59.919 ns_node_disco:4:info:node up - Node 'ns_1@10.1.5.193' saw that node 'ns_1@10.1.5.178' came up. 2011-04-08 16:03:59.962 ns_node_disco:4:info:node up - Node 'ns_1@10.1.5.192' saw that node 'ns_1@10.1.5.178' came up. 2011-04-08 16:04:12.120 menelaus_app:1:info:web start ok - Membase Server has started on web port 8091 on node 'ns_1@10.1.5.178'. 2011-04-08 16:04:12.148 ns_cluster:3:info:message - Node ns_1@10.1.5.178 joined cluster 2011-04-08 16:04:14.383 ns_orchestrator:4:info:message - Starting rebalance, KeepNodes = ['ns_1@10.1.5.192','ns_1@10.1.5.193', 'ns_1@10.1.5.178'], EjectNodes = [] 2011-04-08 16:04:14.666 ns_port_server:0:info:message - Port server memcached on node 'ns_1@10.1.5.178' exited with status 137. Restarting. Messages: 2011-04-08 16:04:14.666 supervisor_cushion:1:warning:port exited too soon after restart - Service memcached exited on node 'ns_1@10.1.5.178' in 2.53s 2011-04-08 16:04:16.534 ns_memcached:1:info:message - Bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" loaded on node 'ns_1@10.1.5.193' in 2 seconds. 2011-04-08 16:04:26.688 ns_memcached:1:info:message - Bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" loaded on node 'ns_1@10.1.5.178' in 1 seconds. 2011-04-08 16:05:00.525 ns_orchestrator:1:info:message - Rebalance completed successfully. 2011-04-08 16:06:59.506 ns_memcached:2:info:message - Shutting down bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" on 'ns_1@10.1.5.193' for server shutdown 2011-04-08 16:06:59.870 ns_orchestrator:6:info:message - Failed over 'ns_1@10.1.5.193': ok 2011-04-08 16:06:59.960 ns_rebalancer:1:info:message - Data has been lost for 33% of vbuckets in bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561". 2011-04-08 16:07:00.085 ns_orchestrator:6:info:message - Failed over 'ns_1@10.1.5.178': ok 2011-04-08 16:07:10.366 ns_memcached:2:info:message - Shutting down bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" on 'ns_1@10.1.5.178' for server shutdown 2011-04-08 16:07:15.094 ns_orchestrator:4:info:message - Starting rebalance, KeepNodes = ['ns_1@10.1.5.192'], EjectNodes = ['ns_1@10.1.5.193', 'ns_1@10.1.5.178'] 2011-04-08 16:07:26.115 ns_orchestrator:2:info:message - Rebalance exited with reason {wait_for_memcached_failed, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", ['ns_1@10.1.5.193','ns_1@10.1.5.178']} logs_node: ------------------------------- INFO REPORT <3.54.0> 2011-04-08 15:58:03 =============================================================================== nonode@nohost:<3.54.0>:log_os_info:25: OS type: {unix,linux} Version: {2,6,18} Runtime info: [{otp_release,"R13B03"}, {erl_version,"5.7.4"}, {erl_version_long, "Erlang R13B03 (erts-5.7.4) [source] [64-bit] [rq:1] [async-threads:16] [kernel-poll:false]\n"}, {system_arch_raw,"x86_64-unknown-linux-gnu"}, {system_arch,"x86_64-unknown-linux-gnu"}, {localtime,{{2011,4,8},{15,58,3}}}, {memory, [{total,15116888}, {processes,1770448}, {processes_used,1758096}, {system,13346440}, {atom,395769}, {atom_used,394478}, {binary,45616}, {code,3475125}, {ets,304744}]}, {loaded, [ns_info,log_os_info,misc,ns_log_mf_h, ns_server_cluster_sup,ns_server,timer,io_lib_fread, cpu_sup,memsup,disksup,os_mon,io_lib_pretty,unicode, io_lib_format,io_lib,io,sasl_report,release_handler, calendar,overload,alarm_handler,sasl_report_tty_h,sasl, ns_bootstrap,file_io_server,orddict,erl_eval,file,c, error_logger_tty_h,queue,kernel_config,user,user_sup, supervisor_bridge,standard_error,gb_sets, hipe_unified_loader,packages,code_server,code,file_server, net_kernel,global_group,erl_distribution,string,ets, filename,inet_gethost_native,inet_parse,inet,inet_udp,os, inet_config,inet_db,global,gb_trees,rpc,dict,supervisor, kernel,application_master,sys,application,gen_server, erl_parse,proplists,erl_scan,lists,application_controller, proc_lib,gen,gen_event,error_logger,heart,error_handler, erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {applications, [{kernel,"ERTS CXC 138 10","2.13.4"}, {stdlib,"ERTS CXC 138 10","1.16.4"}, {os_mon,"CPO CXC 138 46","2.2.4"}, {ns_server,"Membase server","1.6.5.3_291_gba5fe27"}, {menelaus,"Membase menelaus","1.6.5.3_291_gba5fe27"}, {sasl,"SASL CXC 138 11","2.1.8"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,44}, {node,nonode@nohost}, {nodes,[]}, {registered, [overload,init,erl_prim_loader,alarm_handler,error_logger, rex,user,kernel_sup,global_name_server,sasl_sup,inet_db, os_mon_sup,code_server,file_server_2,kernel_safe_sup, cpu_sup,memsup,application_controller,sasl_safe_sup, disksup,standard_error,ns_server_cluster_sup, standard_error_sup,timer_server,global_group, release_handler]}, {cookie,nocookie}, {wordsize,8}, {wall_clock,0}] INFO REPORT <3.55.0> 2011-04-08 15:58:03 =============================================================================== reading ip config from "/etc/membase/priv/ip" PROGRESS REPORT <3.56.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,inet_gethost_native_sup} started [{pid,<3.57.0>},{mfa,{inet_gethost_native,init,[[]]}}] PROGRESS REPORT <3.27.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<3.56.0>}, {name,inet_gethost_native_sup}, {mfa,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] INFO REPORT <3.55.0> 2011-04-08 15:58:03 =============================================================================== nonode@nohost:<3.55.0>:dist_manager:109: Attempting to bring up net_kernel with name 'ns_1@10.1.5.193' PROGRESS REPORT <3.58.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,net_sup} started [{pid,<3.59.0>}, {name,erl_epmd}, {mfa,{erl_epmd,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <3.58.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,net_sup} started [{pid,<3.60.0>}, {name,auth}, {mfa,{auth,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.58.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,net_sup} started [{pid,<0.61.0>}, {name,net_kernel}, {mfa,{net_kernel,start_link,[['ns_1@10.1.5.193',longnames]]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.11.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,kernel_sup} started [{pid,<0.58.0>}, {name,net_sup_dynamic}, {mfa,{erl_distribution,start_link,[['ns_1@10.1.5.193',longnames]]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <0.54.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.55.0>}, {name,dist_manager}, {mfa,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.64.0>}, {name,ns_cluster}, {mfa,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.65.0> 2011-04-08 15:58:03 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<0.66.0>}, {name,mb_mnesia_events}, {mfa,{gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.72.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<0.73.0>}, {name,mnesia_event}, {mfa,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.75.0>}, {name,mnesia_monitor}, {mfa,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.76.0>}, {name,mnesia_subscr}, {mfa,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.77.0>}, {name,mnesia_locker}, {mfa,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.78.0>}, {name,mnesia_recover}, {mfa,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.79.0>}, {name,mnesia_tm}, {mfa,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.80.0>}, {name,mnesia_checkpoint_sup}, {mfa,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.81.0>}, {name,mnesia_snmp_sup}, {mfa,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.82.0>}, {name,mnesia_controller}, {mfa,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.74.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_kernel_sup} started [{pid,<0.83.0>}, {name,mnesia_late_loader}, {mfa,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] PROGRESS REPORT <0.72.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mnesia_sup} started [{pid,<0.74.0>}, {name,mnesia_kernel_sup}, {mfa,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.7.0> 2011-04-08 15:58:04 =============================================================================== application mnesia started_at 'ns_1@10.1.5.193' PROGRESS REPORT <0.27.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.87.0>}, {name,disk_log_sup}, {mfa,{disk_log_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <0.27.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.88.0>}, {name,disk_log_server}, {mfa,{disk_log_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] PROGRESS REPORT <0.27.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.91.0>}, {name,dets_sup}, {mfa,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] PROGRESS REPORT <0.27.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,kernel_safe_sup} started [{pid,<0.92.0>}, {name,dets}, {mfa,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:411: Committed schema to disk. INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:247: Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@10.1.5.193']}, {debug,verbose}, {directory,"/var/membase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,[local_config,cluster,schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@10.1.5.193']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<0.73.0>,<0.67.0>]}, {tables,[local_config,cluster,schema]}, {transaction_commits,5}, {transaction_failures,0}, {transaction_log_writes,3}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {version,"4.4.12"}] Peers: ['ns_1@10.1.5.193'] INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,schema, [{name,schema}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,false}, {record_name,schema}, {attributes,[table,cstruct]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,168798},'ns_1@10.1.5.193'}}, {version,{{3,0},{'ns_1@10.1.5.193',{1302,303484,245769}}}}]}, {tid,3,<0.86.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:194: Peers: ['ns_1@10.1.5.193'] INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:194: Peers: ['ns_1@10.1.5.193'] INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,539692},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,5,<0.111.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,539692},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,5,<0.111.0>}} PROGRESS REPORT <0.65.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,mb_mnesia_sup} started [{pid,<0.67.0>}, {name,mb_mnesia}, {mfa,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.65.0>}, {name,mb_mnesia_sup}, {mfa,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.117.0> 2011-04-08 15:58:04 =============================================================================== loading config from "/etc/membase/priv/config" PROGRESS REPORT <0.117.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.118.0>}, {name,ns_config_events}, {mfa,{gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.117.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.119.0>}, {name,ns_config}, {mfa,{ns_config,start_link, ["/etc/membase/priv/config",ns_config_default]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.117.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.121.0>}, {name,ns_config_isasl_sync}, {mfa,{ns_config_isasl_sync,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== isasl_sync init: ["/var/membase/data/isasl.pw","_admin","_admin"] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== isasl_sync init buckets: [] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== Writing isasl passwd file: "/var/membase/data/isasl.pw" PROGRESS REPORT <0.117.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_config_sup} started [{pid,<0.123.0>}, {name,ns_config_log}, {mfa,{ns_config_log,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.117.0>}, {name,ns_config_sup}, {mfa,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.125.0> 2011-04-08 15:58:04 =============================================================================== ns_log:init(): Couldn't load logs from "/var/membase/data/ns_log": {error, enoent} PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.125.0>}, {name,ns_log}, {mfa,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.126.0>}, {name,ns_log_events}, {mfa,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.127.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.128.0>}, {name,ns_node_disco_events}, {mfa,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.129.0> 2011-04-08 15:58:04 =============================================================================== Initting ns_node_disco with [] INFO REPORT <0.130.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.130.0> 2011-04-08 15:58:04 =============================================================================== ns_log: logging ns_node_disco:3:Initial otp cookie generated: iafrpkdqnfrcgybi INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: otp -> [{cookie,iafrpkdqnfrcgybi}] INFO REPORT <0.130.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.193'], with cookie: iafrpkdqnfrcgybi INFO REPORT <0.130.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.193'], with cookie: iafrpkdqnfrcgybi PROGRESS REPORT <0.127.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.129.0>}, {name,ns_node_disco}, {mfa,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.127.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.134.0>}, {name,ns_node_disco_log}, {mfa,{ns_node_disco_log,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.127.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.135.0>}, {name,ns_node_disco_conf_events}, {mfa,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.136.0> 2011-04-08 15:58:04 =============================================================================== ns_config_rep init pulling INFO REPORT <0.136.0> 2011-04-08 15:58:04 =============================================================================== ns_config_rep init pushing INFO REPORT <0.136.0> 2011-04-08 15:58:04 =============================================================================== ns_config_rep init reannouncing INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: otp -> [{cookie,iafrpkdqnfrcgybi}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: alerts -> [{email,[]}, {email_alerts,false}, [{user,undefined}, {pass,"********"}, {addr,undefined}, {port,undefined}, {encrypt,false}], {alerts,[server_down,server_unresponsive,server_up,server_joined,server_left, bucket_created,bucket_deleted,bucket_auth_failed]}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: buckets -> [{configs,[]}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: memory_quota -> 1570 INFO REPORT <0.138.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.193'] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: port_servers -> [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary","-r", "-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <0.139.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: {node,'ns_1@10.1.5.193',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.138.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.193'], with cookie: iafrpkdqnfrcgybi INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> active INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: {node,'ns_1@10.1.5.193',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: {node,'ns_1@10.1.5.193',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: {node,'ns_1@10.1.5.193',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.139.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.193'], with cookie: iafrpkdqnfrcgybi INFO REPORT <0.138.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.193'], with cookie: iafrpkdqnfrcgybi INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== config change: {node,'ns_1@10.1.5.193',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.139.0> 2011-04-08 15:58:04 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.193'], with cookie: iafrpkdqnfrcgybi INFO REPORT <0.136.0> 2011-04-08 15:58:04 =============================================================================== Pushing config PROGRESS REPORT <0.127.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.136.0>}, {name,ns_config_rep}, {mfa,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.127.0>}, {name,ns_node_disco_sup}, {mfa,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.136.0> 2011-04-08 15:58:04 =============================================================================== Pushing config done INFO REPORT <0.141.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.141.0>:mb_master:83: I'm the only node, so I'm the master. INFO REPORT <0.142.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.142.0>:misc:725: start_singleton(gen_fsm, ns_orchestrator, [], []): started as <0.143.0> on 'ns_1@10.1.5.193' PROGRESS REPORT <0.142.0> 2011-04-08 15:58:04 =============================================================================== supervisor {<0.142.0>,mb_master_sup} started [{pid,<0.143.0>}, {name,ns_orchestrator}, {mfa,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] INFO REPORT <0.142.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.142.0>:misc:725: start_singleton(gen_server, ns_tick, [], []): started as <0.144.0> on 'ns_1@10.1.5.193' PROGRESS REPORT <0.142.0> 2011-04-08 15:58:04 =============================================================================== supervisor {<0.142.0>,mb_master_sup} started [{pid,<0.144.0>}, {name,ns_tick}, {mfa,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.141.0>}, {name,mb_master}, {mfa,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.145.0>}, {name,ns_heart}, {mfa,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.147.0>}, {name,ns_doctor}, {mfa,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.147.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.147.0>:ns_doctor:82: Got initial status [{'ns_1@10.1.5.193', [{last_heard, {1302,303484, 763789}}, {active_buckets, []}, {memory, [{total, 20174504}, {processes, 3064928}, {processes_used, 3058664}, {system, 17109576}, {atom,649969}, {atom_used, 635345}, {binary,36904}, {code,6739010}, {ets,483368}]}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.13.4"}, {mnesia, "4.4.12"}, {stdlib, "1.16.4"}, {os_mon, "2.2.4"}, {ns_server, "1.6.5.3_291_gba5fe27"}, {menelaus, "1.6.5.3_291_gba5fe27"}, {sasl, "2.1.8"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,1}, {memory_data, {2058928128, 984981504, {<0.3.0>, 425800}}}, {disk_data, [{"/",13140720, 39}, {"/boot", 101086,13}, {"/dev/shm", 1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1043916 kB\nBuffers: 252616 kB\nCached: 504592 kB\nSwapCached: 17896 kB\nActive: 599728 kB\nInactive: 230948 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1043916 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 194108 kB\nWriteback: 0 kB\nAnonPages: 68936 kB\nMapped: 18664 kB\nSlab: 113388 kB\nPageTables: 5472 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 221116 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory, 2058928128}, {free_swap, 2031652864}, {total_swap, 2080366592}, {cached_memory, 516702208}, {buffered_memory, 258678784}, {free_memory, 1068969984}, {total_memory, 2058928128}]}, {statistics, [{wall_clock, {1669,888}}, {context_switches, {7868,0}}, {garbage_collection, {1297, 2801269,0}}, {io, {{input, 3602360}, {output, 194075}}}, {reductions, {1281337, 1281337}}, {run_queue,0}, {runtime, {340, 340}}]}]}] PROGRESS REPORT <0.149.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.150.0>}, {name,menelaus_web}, {mfa,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.149.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.152.0>}, {name,menelaus_event}, {mfa,{menelaus_event,start_link,[]}}, {restart_type,transient}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.149.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.153.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.149.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.154.0>}, {name,menelaus_web_alerts_srv}, {mfa,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== ns_log: logging menelaus_app:1:Membase Server has started on web port 8091 on node 'ns_1@10.1.5.193'. PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.149.0>}, {name,menelaus}, {mfa,{menelaus_app,start_subapp,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.155.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.156.0>}, {name,ns_port_init}, {mfa,{ns_port_init,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.157.0> 2011-04-08 15:58:04 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.155.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.157.0>}, {name, {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.159.0> 2011-04-08 15:58:04 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.155.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.159.0>}, {name, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p","11210", "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.155.0>}, {name,ns_port_sup}, {mfa,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.161.0>}, {name,ns_tick_event}, {mfa,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.162.0>}, {name,ns_stats_event}, {mfa,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.163.0>}, {name,ns_bucket_worker}, {mfa,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.164.0>}, {name,ns_bucket_sup}, {mfa,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.165.0>}, {name,system_stats_collector}, {mfa,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.166.0>}, {name,{stats_archiver,"@system"}}, {mfa,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-minute' PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.171.0>}, {name,{stats_reader,"@system"}}, {mfa,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.172.0>}, {name,ns_moxi_sup}, {mfa,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.173.0> 2011-04-08 15:58:04 =============================================================================== starting moxi_stats_collector with delay of 5000 PROGRESS REPORT <0.124.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.173.0>}, {name,moxi_stats_collector}, {mfa,{supervisor_cushion,start_link, [moxi_stats_collector,5000, moxi_stats_collector,start_link,[]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 15:58:04 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.124.0>}, {name,ns_server_sup}, {mfa,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.7.0> 2011-04-08 15:58:04 =============================================================================== application ns_server started_at 'ns_1@10.1.5.193' INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,830881},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,6,<0.168.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,830881},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,6,<0.168.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-hour' INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,866633},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,7,<0.180.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,866633},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,7,<0.180.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-day' INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,881872},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,8,<0.187.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,881872},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,8,<0.187.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-week' INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,888217},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,9,<0.194.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,888217},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,9,<0.194.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-month' INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,904851},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,10,<0.201.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,904851},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,10,<0.201.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-@system-year' INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,909795},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,11,<0.208.0>}} INFO REPORT <0.67.0> 2011-04-08 15:58:04 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303484,909795},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,11,<0.208.0>}} INFO REPORT <0.118.0> 2011-04-08 15:58:09 =============================================================================== unsupervising port: {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,stderr_to_stdout,stream]} INFO REPORT <0.118.0> 2011-04-08 15:58:09 =============================================================================== supervising port: {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]} INFO REPORT <0.216.0> 2011-04-08 15:58:09 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.155.0> 2011-04-08 15:58:09 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.216.0>}, {name, {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.118.0> 2011-04-08 15:58:09 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 15:58:09 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.136.0> 2011-04-08 15:58:09 =============================================================================== Pushing config INFO REPORT <0.136.0> 2011-04-08 15:58:09 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 15:58:09 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.118.0> 2011-04-08 15:58:09 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.136.0> 2011-04-08 15:58:09 =============================================================================== Pushing config INFO REPORT <0.136.0> 2011-04-08 15:58:09 =============================================================================== Pushing config done INFO REPORT <0.151.0> 2011-04-08 15:58:09 =============================================================================== menelaus_web streaming socket closed by client INFO REPORT <0.217.0> 2011-04-08 15:58:10 =============================================================================== moxi<0.217.0>: 2011-04-08 15:58:09: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13) moxi<0.217.0>: 2011-04-08 15:58:09: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <0.147.0> 2011-04-08 15:59:04 =============================================================================== ns_1@10.1.5.193:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.193', [{last_heard,{1302,303544,758228}}, {active_buckets,[]}, {memory, [{total,23766240}, {processes,5595880}, {processes_used,5587584}, {system,18170360}, {atom,728913}, {atom_used,696147}, {binary,45328}, {code,7538993}, {ets,637496}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,61}, {memory_data,{2058928128,993116160,{<0.124.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1040832 kB\nBuffers: 252620 kB\nCached: 504720 kB\nSwapCached: 17896 kB\nActive: 605540 kB\nInactive: 229608 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1040832 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 55520 kB\nWriteback: 0 kB\nAnonPages: 73248 kB\nMapped: 18384 kB\nSlab: 111328 kB\nPageTables: 5632 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 371032 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,516833280}, {buffered_memory,258682880}, {free_memory,1065811968}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{51669,1}}, {context_switches,{13408,0}}, {garbage_collection,{2601,4714050,0}}, {io,{{input,4095488},{output,320919}}}, {reductions,{2124099,87866}}, {run_queue,0}, {runtime,{490,10}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:00:04 =============================================================================== ns_1@10.1.5.193:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.193', [{last_heard,{1302,303604,758244}}, {active_buckets,[]}, {memory, [{total,23796528}, {processes,5580088}, {processes_used,5571792}, {system,18216440}, {atom,728913}, {atom_used,696147}, {binary,55016}, {code,7538993}, {ets,672304}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,121}, {memory_data,{2058928128,993243136,{<0.124.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1040708 kB\nBuffers: 252620 kB\nCached: 504800 kB\nSwapCached: 17896 kB\nActive: 605656 kB\nInactive: 229612 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1040708 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 43204 kB\nWriteback: 0 kB\nAnonPages: 73288 kB\nMapped: 18384 kB\nSlab: 111088 kB\nPageTables: 5632 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 371960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,516915200}, {buffered_memory,258682880}, {free_memory,1065684992}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{111669,1}}, {context_switches,{17586,0}}, {garbage_collection,{3423,5889293,0}}, {io,{{input,4107144},{output,404112}}}, {reductions,{2638760,104617}}, {run_queue,0}, {runtime,{580,10}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:01:04 =============================================================================== ns_1@10.1.5.193:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.193', [{last_heard,{1302,303664,758233}}, {active_buckets,[]}, {memory, [{total,23848848}, {processes,5602384}, {processes_used,5592488}, {system,18246464}, {atom,728913}, {atom_used,696147}, {binary,47512}, {code,7538993}, {ets,707392}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,181}, {memory_data,{2058928128,993370112,{<0.124.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1040584 kB\nBuffers: 252620 kB\nCached: 504884 kB\nSwapCached: 17896 kB\nActive: 605768 kB\nInactive: 229616 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1040584 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 34912 kB\nWriteback: 0 kB\nAnonPages: 73324 kB\nMapped: 18384 kB\nSlab: 111052 kB\nPageTables: 5632 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 371960 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,517001216}, {buffered_memory,258682880}, {free_memory,1065558016}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{171669,1}}, {context_switches,{21778,0}}, {garbage_collection,{4267,7040217,0}}, {io,{{input,4118800},{output,489132}}}, {reductions,{3164773,91881}}, {run_queue,0}, {runtime,{680,10}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:02:04 =============================================================================== ns_1@10.1.5.193:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.193', [{last_heard,{1302,303724,758182}}, {active_buckets,[]}, {memory, [{total,23854928}, {processes,5580080}, {processes_used,5570040}, {system,18274848}, {atom,728913}, {atom_used,696147}, {binary,43296}, {code,7538993}, {ets,738456}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,241}, {memory_data,{2058928128,993624064,{<0.124.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1040336 kB\nBuffers: 252624 kB\nCached: 504948 kB\nSwapCached: 17896 kB\nActive: 605860 kB\nInactive: 229784 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1040336 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 26532 kB\nWriteback: 0 kB\nAnonPages: 73516 kB\nMapped: 18384 kB\nSlab: 111080 kB\nPageTables: 5632 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 371032 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,517066752}, {buffered_memory,258686976}, {free_memory,1065304064}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{231669,1}}, {context_switches,{26333,0}}, {garbage_collection,{5295,8290295,0}}, {io,{{input,4375004},{output,786856}}}, {reductions,{3710466,83831}}, {run_queue,0}, {runtime,{780,10}}]}]}] INFO REPORT <0.147.0> 2011-04-08 16:03:04 =============================================================================== ns_1@10.1.5.193:<0.147.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.193', [{last_heard,{1302,303784,758253}}, {active_buckets,[]}, {memory, [{total,23926512}, {processes,5618400}, {processes_used,5608504}, {system,18308112}, {atom,728913}, {atom_used,696147}, {binary,43680}, {code,7538993}, {ets,769824}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,301}, {memory_data,{2058928128,993751040,{<0.124.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1040212 kB\nBuffers: 252624 kB\nCached: 505028 kB\nSwapCached: 17896 kB\nActive: 605980 kB\nInactive: 229788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1040212 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 18316 kB\nWriteback: 0 kB\nAnonPages: 73552 kB\nMapped: 18384 kB\nSlab: 111084 kB\nPageTables: 5632 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 371032 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication,[]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,517148672}, {buffered_memory,258686976}, {free_memory,1065177088}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{291669,1}}, {context_switches,{30543,0}}, {garbage_collection,{6119,9445437,0}}, {io,{{input,4386660},{output,873080}}}, {reductions,{4219018,103633}}, {run_queue,0}, {runtime,{880,10}}]}]}] INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== ********* RB: UNPRINTABLE REPORT ******** INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== ns_1@10.1.5.193:<0.64.0>:ns_cluster:98: engage_cluster(..) -> {ok,ok} INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== ns_1@10.1.5.193:<0.64.0>:ns_cluster:102: handling complete_join([{<<"targetNode">>, <<"ns_1@10.1.5.193">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 13140720}, {<<"usagePercent">>, 42}]}, {struct, [{<<"path">>, <<"/boot">>}, {<<"sizeKBytes">>, 101086}, {<<"usagePercent">>, 13}]}, {struct, [{<<"path">>, <<"/dev/shm">>}, {<<"sizeKBytes">>, 1005336}, {<<"usagePercent">>, 0}]}]}]}}, {<<"memoryQuota">>, 1570}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 26971062}, {<<"total">>, 2058928128}, {<<"quotaTotal">>, 1646264320}, {<<"used">>, 1567481856}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 2828944}, {<<"total">>, 13456097280.0}, {<<"quotaTotal">>, 13456097280.0}, {<<"used">>, 5651560857.0}, {<<"free">>, 7804536423.0}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>, []}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/var/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"uptime">>, <<"375">>}, {<<"memoryTotal">>, 2058928128}, {<<"memoryFree">>, 491446272}, {<<"mcdMemoryReserved">>, 1570}, {<<"mcdMemoryAllocated">>, 1570}, {<<"otpNode">>, <<"ns_1@10.1.5.192">>}, {<<"otpCookie">>, <<"lulxzmudqfmlvwvs">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.1.5.192:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.6.5.3_291_gba5fe27">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>, 11211}, {<<"direct">>, 11210}]}}]) INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== ns_log: logging ns_cluster:2:Node 'ns_1@10.1.5.193' is joining cluster via node 'ns_1@10.1.5.192'. INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: i_am_a_dead_man -> true ERROR REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ** gen_event handler {ns_pubsub,#Ref<0.0.0.680>} crashed. ** Was installed in ns_config_events ** Last event was: [{i_am_a_dead_man,true}] ** When handler state == {state,#Fun,ok} ** Reason == {noproc,{gen_server,call,[ns_moxi_sup,which_children,infinity]}} ERROR REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ** gen_event handler ns_port_init crashed. ** Was installed in ns_config_events ** Last event was: [{i_am_a_dead_man,true}] ** When handler state == {state} ** Reason == {noproc,{gen_server,call,[ns_port_sup,which_children,infinity]}} INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.136.0> 2011-04-08 16:03:47 =============================================================================== Pushing config INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== ns_cluster: joining cluster. Child has exited. INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: otp -> [{cookie,lulxzmudqfmlvwvs}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.193','ns_1@10.1.5.192'] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== pre-join cleaned config is: {config, {full,"/etc/membase/priv/config",undefined,ns_config_default}, [[{directory,"/var/membase/config"}], [{directory,"/opt/membase/1.7.0r/config"}, {nodes_wanted,['ns_1@10.1.5.193']}, {{node,'ns_1@10.1.5.193',membership},active}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@10.1.5.193',isasl},[{path,"/var/membase/data/isasl.pw"}]}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {memory_quota,1570}, {buckets,[{configs,[]}]}, {{node,'ns_1@10.1.5.193',moxi},[{port,11211},{verbosity,[]}]}, {port_servers, [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {alerts, [{email,[]}, {email_alerts,false}, {email_server, [{user,undefined}, {pass,undefined}, {addr,undefined}, {port,undefined}, {encrypt,false}]}, {alerts, [server_down,server_unresponsive,server_up,server_joined, server_left,bucket_created,bucket_deleted, bucket_auth_failed]}]}, {replication,[{enabled,true}]}]], [[{nodes_wanted,['ns_1@10.1.5.193','ns_1@10.1.5.192']}, {{node,'ns_1@10.1.5.193',rest},[{port,8091}]}, {{node,'ns_1@10.1.5.193',ns_log}, [{filename,"/var/membase/data/ns_log"}]}, {{node,'ns_1@10.1.5.193',moxi},[{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.5.193',memcached}, [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines, [{membase, [{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.5.193',isasl},[{path,"/var/membase/data/isasl.pw"}]}, {otp, [{'_vclock',[{'ns_1@10.1.5.193',{1,63469523027}}]}, {cookie,lulxzmudqfmlvwvs}]}]], ns_config_default, {ns_config,save_config_sync,[]}, <0.420.0>,true} INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== ns_1@10.1.5.193:<0.64.0>:ns_cluster:572: Connection from 'ns_1@10.1.5.193' to 'ns_1@10.1.5.192': true INFO REPORT <0.64.0> 2011-04-08 16:03:47 =============================================================================== ns_1@10.1.5.193:<0.64.0>:ns_cluster:581: Join status: {ok,ok}, starting ns_server_cluster back PROGRESS REPORT <0.427.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.428.0>}, {name,ns_log}, {mfa,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.429.0>}, {name,ns_log_events}, {mfa,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.430.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.431.0>}, {name,ns_node_disco_events}, {mfa,{gen_event,start_link,[{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.432.0> 2011-04-08 16:03:47 =============================================================================== Initting ns_node_disco with ['ns_1@10.1.5.192'] INFO REPORT <0.433.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.433.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.433.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs PROGRESS REPORT <0.430.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.432.0>}, {name,ns_node_disco}, {mfa,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.430.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.437.0>}, {name,ns_node_disco_log}, {mfa,{ns_node_disco_log,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.430.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.438.0>}, {name,ns_node_disco_conf_events}, {mfa,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== ns_config_rep init pulling INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: alerts -> [{email,[]}, {email_alerts,false}, [{user,undefined}, {pass,"********"}, {addr,undefined}, {port,undefined}, {encrypt,false}], {alerts,[server_down,server_unresponsive,server_up,server_joined,server_left, bucket_created,bucket_deleted,bucket_auth_failed]}] INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== ns_config_rep init pushing INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== Writing isasl passwd file: "/var/membase/data/isasl.pw" INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: memory_quota -> 1570 INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.192','ns_1@10.1.5.193'] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: port_servers -> [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary","-r", "-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <0.444.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.445.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== ns_config_rep init reannouncing INFO REPORT <0.444.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.445.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs PROGRESS REPORT <0.430.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_node_disco_sup} started [{pid,<0.439.0>}, {name,ns_config_rep}, {mfa,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.430.0>}, {name,ns_node_disco_sup}, {mfa,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> inactiveAdded INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.444.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.445.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: alerts -> [{email,[]}, {email_alerts,false}, [{user,undefined}, {pass,"********"}, {addr,undefined}, {port,undefined}, {encrypt,false}], {alerts,[server_down,server_unresponsive,server_up,server_joined,server_left, bucket_created,bucket_deleted,bucket_auth_failed]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: memory_quota -> 1570 INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.192','ns_1@10.1.5.193'] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config on otp INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: otp -> [{cookie,lulxzmudqfmlvwvs}] INFO REPORT <0.451.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: port_servers -> [{moxi,"./bin/moxi/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{rest,port}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,stderr_to_stdout,stream]}, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", {"~B",[port]}, "-E","./bin/bucket_engine/bucket_engine.so","-B","binary","-r", "-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: replication -> [{enabled,true}] INFO REPORT <0.452.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.453.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.451.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.454.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.452.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.453.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.454.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.451.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.452.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.453.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.454.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.192','ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: rest -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: rest_creds -> ******** INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.192',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.193',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> inactiveAdded INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.193',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.193',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.193',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== config change: {node,'ns_1@10.1.5.193',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:03:47 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:47 =============================================================================== Pushing config done INFO REPORT <0.446.0> 2011-04-08 16:03:47 =============================================================================== ns_1@10.1.5.193:<0.446.0>:mb_master:93: Starting as candidate. Peers: ['ns_1@10.1.5.192', 'ns_1@10.1.5.193'] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.446.0>}, {name,mb_master}, {mfa,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.459.0>}, {name,ns_heart}, {mfa,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:47 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.460.0>}, {name,ns_doctor}, {mfa,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.446.0> 2011-04-08 16:03:47 =============================================================================== ns_1@10.1.5.193:<0.446.0>:mb_master:196: Changing master from undefined to 'ns_1@10.1.5.192' PROGRESS REPORT <0.462.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.463.0>}, {name,menelaus_web}, {mfa,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.462.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.465.0>}, {name,menelaus_event}, {mfa,{menelaus_event,start_link,[]}}, {restart_type,transient}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.462.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.466.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] PROGRESS REPORT <0.462.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.467.0>}, {name,menelaus_web_alerts_srv}, {mfa,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== ns_log: logging menelaus_app:1:Membase Server has started on web port 8091 on node 'ns_1@10.1.5.193'. PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.462.0>}, {name,menelaus}, {mfa,{menelaus_app,start_subapp,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.468.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.469.0>}, {name,ns_port_init}, {mfa,{ns_port_init,start_link,[]}}, {restart_type,transient}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.470.0> 2011-04-08 16:03:48 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.468.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.470.0>}, {name, {moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.472.0> 2011-04-08 16:03:48 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.468.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.472.0>}, {name, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p","11210", "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.468.0>}, {name,ns_port_sup}, {mfa,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.474.0>}, {name,ns_tick_event}, {mfa,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.475.0>}, {name,ns_stats_event}, {mfa,{gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.476.0>}, {name,ns_bucket_worker}, {mfa,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.477.0>}, {name,ns_bucket_sup}, {mfa,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.479.0>}, {name,system_stats_collector}, {mfa,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.480.0>}, {name,{stats_archiver,"@system"}}, {mfa,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.481.0>}, {name,{stats_reader,"@system"}}, {mfa,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.482.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_moxi_sup} started [{pid,<0.483.0>}, {name,{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",[], 11220,8091}}, {mfa,{ns_port_server,start_link, [moxi,"./bin/moxi/moxi", ["-B","auto","-z", "url=http://127.0.0.1:8091/pools/default/bucketsStreaming/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", "-Z", "port_listen=11220,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-p","0","-Y","y","-O","stderr"], [use_stdio,stderr_to_stdout, {env,[{"MOXI_SASL_PLAIN_USR", "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {"MOXI_SASL_PLAIN_PWD",[]}]}]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.482.0>}, {name,ns_moxi_sup}, {mfa,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.484.0> 2011-04-08 16:03:48 =============================================================================== starting moxi_stats_collector with delay of 5000 PROGRESS REPORT <0.427.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.484.0>}, {name,moxi_stats_collector}, {mfa,{supervisor_cushion,start_link, [moxi_stats_collector,5000, moxi_stats_collector,start_link,[]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.54.0> 2011-04-08 16:03:48 =============================================================================== supervisor {local,ns_server_cluster_sup} started [{pid,<0.427.0>}, {name,ns_server_sup}, {mfa,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.64.0> 2011-04-08 16:03:48 =============================================================================== ns_log: logging ns_cluster:3:Node ns_1@10.1.5.193 joined cluster INFO REPORT <0.64.0> 2011-04-08 16:03:48 =============================================================================== ns_1@10.1.5.193:<0.64.0>:ns_cluster:104: complete_join([{<<"targetNode">>, <<"ns_1@10.1.5.193">>}, {<<"availableStorage">>, {struct, [{<<"hdd">>, [{struct, [{<<"path">>, <<"/">>}, {<<"sizeKBytes">>, 13140720}, {<<"usagePercent">>, 42}]}, {struct, [{<<"path">>, <<"/boot">>}, {<<"sizeKBytes">>, 101086}, {<<"usagePercent">>, 13}]}, {struct, [{<<"path">>, <<"/dev/shm">>}, {<<"sizeKBytes">>, 1005336}, {<<"usagePercent">>, 0}]}]}]}}, {<<"memoryQuota">>, 1570}, {<<"storageTotals">>, {struct, [{<<"ram">>, {struct, [{<<"usedByData">>, 26971062}, {<<"total">>, 2058928128}, {<<"quotaTotal">>, 1646264320}, {<<"used">>, 1567481856}]}}, {<<"hdd">>, {struct, [{<<"usedByData">>, 2828944}, {<<"total">>, 13456097280.0}, {<<"quotaTotal">>, 13456097280.0}, {<<"used">>, 5651560857.0}, {<<"free">>, 7804536423.0}]}}]}}, {<<"storage">>, {struct, [{<<"ssd">>,[]}, {<<"hdd">>, [{struct, [{<<"path">>, <<"/var/membase/data">>}, {<<"quotaMb">>, <<"none">>}, {<<"state">>, <<"ok">>}]}]}]}}, {<<"uptime">>, <<"375">>}, {<<"memoryTotal">>, 2058928128}, {<<"memoryFree">>, 491446272}, {<<"mcdMemoryReserved">>, 1570}, {<<"mcdMemoryAllocated">>, 1570}, {<<"otpNode">>, <<"ns_1@10.1.5.192">>}, {<<"otpCookie">>, <<"lulxzmudqfmlvwvs">>}, {<<"clusterMembership">>, <<"active">>}, {<<"status">>, <<"healthy">>}, {<<"hostname">>, <<"10.1.5.192:8091">>}, {<<"clusterCompatibility">>, 1}, {<<"version">>, <<"1.6.5.3_291_gba5fe27">>}, {<<"os">>, <<"x86_64-unknown-linux-gnu">>}, {<<"ports">>, {struct, [{<<"proxy">>,11211}, {<<"direct">>, 11210}]}}]) -> {ok, ok} INFO REPORT <0.460.0> 2011-04-08 16:03:48 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:82: Got initial status [{'ns_1@10.1.5.192', [{last_heard, {1302,303828, 489779}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total, 30200312}, {processes, 10204408}, {processes_used, 10184136}, {system, 19995904}, {atom,820249}, {atom_used, 812592}, {binary, 177352}, {code,8534778}, {ets, 1143040}]}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.13.4"}, {mnesia, "4.4.12"}, {stdlib, "1.16.4"}, {os_mon, "2.2.4"}, {inets,"5.2"}, {ns_server, "1.6.5.3_291_gba5fe27"}, {menelaus, "1.6.5.3_291_gba5fe27"}, {sasl, "2.1.8"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,377}, {memory_data, {2058928128, 1567481856, {<10461.124.0>, 1572784}}}, {disk_data, [{"/",13140720, 42}, {"/boot", 101086,13}, {"/dev/shm", 1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 477944 kB\nBuffers: 352548 kB\nCached: 893556 kB\nSwapCached: 17552 kB\nActive: 754340 kB\nInactive: 629496 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 477944 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 11468 kB\nWriteback: 0 kB\nAnonPages: 133444 kB\nMapped: 17492 kB\nSlab: 124320 kB\nPageTables: 6608 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 565260 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", 1.0}]}, {system_memory_data, [{system_total_memory, 2058928128}, {free_swap, 2014429184}, {total_swap, 2080366592}, {cached_memory, 915001344}, {buffered_memory, 361009152}, {free_memory, 489414656}, {total_memory, 2058928128}]}, {statistics, [{wall_clock, {371308,86}}, {context_switches, {52072,0}}, {garbage_collection, {10833, 24525653,0}}, {io, {{input, 5264749}, {output, 2264320}}}, {reductions, {12988877, 2033270}}, {run_queue,0}, {runtime, {1940, 180}}]}]}, {'ns_1@10.1.5.193', [{last_heard, {1302,303828, 489749}}, {active_buckets, []}, {memory, [{total, 25359096}, {processes, 6923872}, {processes_used, 6913592}, {system, 18435224}, {atom,730529}, {atom_used, 698531}, {binary,67064}, {code,7582094}, {ets,793192}]}, {cluster_compatibility_version, 1}, {version, [{kernel, "2.13.4"}, {mnesia, "4.4.12"}, {stdlib, "1.16.4"}, {os_mon, "2.2.4"}, {ns_server, "1.6.5.3_291_gba5fe27"}, {menelaus, "1.6.5.3_291_gba5fe27"}, {sasl, "2.1.8"}]}, {system_arch, "x86_64-unknown-linux-gnu"}, {wall_clock,345}, {memory_data, {2058928128, 993751040, {<0.124.0>, 1572784}}}, {disk_data, [{"/",13140720, 39}, {"/boot", 101086,13}, {"/dev/shm", 1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 1038392 kB\nBuffers: 252624 kB\nCached: 505164 kB\nSwapCached: 17896 kB\nActive: 608484 kB\nInactive: 229780 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 1038392 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 14276 kB\nWriteback: 0 kB\nAnonPages: 75888 kB\nMapped: 18128 kB\nSlab: 110988 kB\nPageTables: 5632 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 342052 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", 1.0}]}, {system_memory_data, [{system_total_memory, 2058928128}, {free_swap, 2031652864}, {total_swap, 2080366592}, {cached_memory, 517287936}, {buffered_memory, 258686976}, {free_memory, 1063448576}, {total_memory, 2058928128}]}, {statistics, [{wall_clock, {345084,640}}, {context_switches, {35703,0}}, {garbage_collection, {7075, 11350078,0}}, {io, {{input, 4484560}, {output, 1040821}}}, {reductions, {5058011, 375627}}, {run_queue,1}, {runtime, {1050, 80}}]}]}] CRASH REPORT <0.485.0> 2011-04-08 16:03:48 =============================================================================== Crashing process initial_call {moxi_stats_collector,init,['Argument__1']} pid <0.485.0> registered_name [] error_info {exit,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}, [{gen_server,init_it,6},{proc_lib,init_p_do_apply,3}]} ancestors [<0.484.0>,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [] links [<0.484.0>] dictionary [] trap_exit false status running heap_size 17711 stack_size 24 reductions 339 INFO REPORT <0.484.0> 2011-04-08 16:03:48 =============================================================================== Cushion got unexpected info supervising moxi_stats_collector: {ack,<0.485.0>, {error, {{badmatch, {error, econnrefused}}, [{moxi_stats_collector, connect_moxi, 1}, {moxi_stats_collector, init,1}, {gen_server, init_it,6}, {proc_lib, init_p_do_apply, 3}]}}} INFO REPORT <0.484.0> 2011-04-08 16:03:48 =============================================================================== ns_log: logging supervisor_cushion:1:Service moxi_stats_collector exited on node 'ns_1@10.1.5.193' in 0.20s INFO REPORT <0.484.0> 2011-04-08 16:03:48 =============================================================================== Cushion managed supervisor for moxi_stats_collector failed: {{badmatch, {error, econnrefused}}, [{moxi_stats_collector, connect_moxi, 1}, {moxi_stats_collector, init,1}, {gen_server, init_it,6}, {proc_lib, init_p_do_apply, 3}]} INFO REPORT <0.484.0> 2011-04-08 16:03:48 =============================================================================== ns_log: logging supervisor_cushion:1:Service moxi_stats_collector exited on node 'ns_1@10.1.5.193' in 0.20s INFO REPORT <0.471.0> 2011-04-08 16:03:49 =============================================================================== moxi<0.471.0>: 2011-04-08 16:03:48: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (13) moxi<0.471.0>: 2011-04-08 16:03:48: (cproxy_config.c.325) env: MOXI_SASL_PLAIN_PWD (8) INFO REPORT <0.483.0> 2011-04-08 16:03:49 =============================================================================== moxi<0.483.0>: 2011-04-08 16:03:48: (cproxy_config.c.316) env: MOXI_SASL_PLAIN_USR (55) INFO REPORT <0.473.0> 2011-04-08 16:03:49 =============================================================================== ns_log: logging ns_port_server:0:Port server memcached on node 'ns_1@10.1.5.193' exited with status 137. Restarting. Messages: ERROR REPORT <0.473.0> 2011-04-08 16:03:49 =============================================================================== ** Generic server <0.473.0> terminating ** Last message in was {#Port<0.2544>,{exit_status,137}} ** When Server state == {state,#Port<0.2544>,memcached, {[empty],[empty,empty]}, undefined,[],0} ** Reason for termination == ** {abnormal,137} CRASH REPORT <0.473.0> 2011-04-08 16:03:49 =============================================================================== Crashing process initial_call {ns_port_server,init,['Argument__1']} pid <0.473.0> registered_name [] error_info {exit,{abnormal,137}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [<0.472.0>,ns_port_sup,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [{'EXIT',#Port<0.2544>,normal}] links [<0.472.0>] dictionary [] trap_exit true status running heap_size 610 stack_size 24 reductions 689 INFO REPORT <0.472.0> 2011-04-08 16:03:49 =============================================================================== Cushion managed supervisor for memcached failed: {abnormal,137} INFO REPORT <0.472.0> 2011-04-08 16:03:49 =============================================================================== ns_log: logging supervisor_cushion:1:Service memcached exited on node 'ns_1@10.1.5.193' in 1.67s ERROR REPORT <0.484.0> 2011-04-08 16:03:53 =============================================================================== ** Generic server <0.484.0> terminating ** Last message in was {die, {error,cushioned_supervisor, {ack,<0.485.0>, {error, {{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}}} ** When Server state == {state,moxi_stats_collector,5000, {1302,303828,289611}, undefined} ** Reason for termination == ** {error,cushioned_supervisor, {ack,<0.485.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}} CRASH REPORT <0.484.0> 2011-04-08 16:03:53 =============================================================================== Crashing process initial_call {supervisor_cushion,init,['Argument__1']} pid <0.484.0> registered_name [] error_info {exit,{error,cushioned_supervisor, {ack,<0.485.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [{die,{error,cushioned_supervisor, {{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}] links [<0.427.0>] dictionary [] trap_exit true status running heap_size 987 stack_size 24 reductions 1101 SUPERVISOR REPORT <0.427.0> 2011-04-08 16:03:53 =============================================================================== Reporting supervisor {local,ns_server_sup} Child process errorContext child_terminated reason {error,cushioned_supervisor, {ack,<0.485.0>, {error,{{badmatch,{error,econnrefused}}, [{moxi_stats_collector,connect_moxi,1}, {moxi_stats_collector,init,1}, {gen_server,init_it,6}, {proc_lib,init_p_do_apply,3}]}}}} pid <0.484.0> name moxi_stats_collector start_function {supervisor_cushion,start_link, [moxi_stats_collector,5000,moxi_stats_collector, start_link,[]]} restart_type permanent shutdown 10 child_type worker INFO REPORT <0.490.0> 2011-04-08 16:03:53 =============================================================================== starting moxi_stats_collector with delay of 5000 PROGRESS REPORT <0.427.0> 2011-04-08 16:03:53 =============================================================================== supervisor {local,ns_server_sup} started [{pid,<0.490.0>}, {name,moxi_stats_collector}, {mfa,{supervisor_cushion,start_link, [moxi_stats_collector,5000, moxi_stats_collector,start_link,[]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] ERROR REPORT <0.472.0> 2011-04-08 16:03:54 =============================================================================== ** Generic server <0.472.0> terminating ** Last message in was {die,{error,cushioned_supervisor,{abnormal,137}}} ** When Server state == {state,memcached,5000,{1302,303828,167664},undefined} ** Reason for termination == ** {error,cushioned_supervisor,{abnormal,137}} CRASH REPORT <0.472.0> 2011-04-08 16:03:54 =============================================================================== Crashing process initial_call {supervisor_cushion,init,['Argument__1']} pid <0.472.0> registered_name [] error_info {exit,{error,cushioned_supervisor,{abnormal,137}}, [{gen_server,terminate,6},{proc_lib,init_p_do_apply,3}]} ancestors [ns_port_sup,ns_server_sup,ns_server_cluster_sup,<0.53.0>] messages [] links [<0.468.0>] dictionary [] trap_exit true status running heap_size 377 stack_size 24 reductions 559 SUPERVISOR REPORT <0.468.0> 2011-04-08 16:03:54 =============================================================================== Reporting supervisor {local,ns_port_sup} Child process errorContext child_terminated reason {error,cushioned_supervisor,{abnormal,137}} pid <0.472.0> name {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]} start_function {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so", "-p","11210","-E", "./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]} restart_type permanent shutdown 10 child_type worker INFO REPORT <0.492.0> 2011-04-08 16:03:54 =============================================================================== starting ns_port_server with delay of 5000 PROGRESS REPORT <0.468.0> 2011-04-08 16:03:54 =============================================================================== supervisor {local,ns_port_sup} started [{pid,<0.492.0>}, {name, {memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p","11210", "-E","./bin/bucket_engine/bucket_engine.so","-B","binary", "-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]}}, {mfa, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached,"./bin/memcached", ["-X","./lib/memcached/stdin_term_handler.so","-p", "11210","-E","./bin/bucket_engine/bucket_engine.so","-B", "binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE","/var/membase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco_conf_events config on nodes_wanted INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== config change: nodes_wanted -> ['ns_1@10.1.5.178','ns_1@10.1.5.192','ns_1@10.1.5.193'] INFO REPORT <0.446.0> 2011-04-08 16:03:59 =============================================================================== ns_1@10.1.5.193:<0.446.0>:mb_master:310: List of peers has changed from ['ns_1@10.1.5.192', 'ns_1@10.1.5.193'] to ['ns_1@10.1.5.178', 'ns_1@10.1.5.192', 'ns_1@10.1.5.193'] INFO REPORT <0.498.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.499.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco cookie_sync INFO REPORT <0.498.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.499.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco: nodes_wanted updated: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> inactiveAdded INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:03:59 =============================================================================== Pushing config done INFO REPORT <0.432.0> 2011-04-08 16:03:59 =============================================================================== ns_log: logging ns_node_disco:4:Node 'ns_1@10.1.5.193' saw that node 'ns_1@10.1.5.178' came up. INFO REPORT <0.498.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.499.0> 2011-04-08 16:03:59 =============================================================================== ns_node_disco: nodes_wanted pong: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'], with cookie: lulxzmudqfmlvwvs INFO REPORT <0.118.0> 2011-04-08 16:04:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',isasl} -> [{path,"/var/membase/data/isasl.pw"}] INFO REPORT <0.118.0> 2011-04-08 16:04:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',memcached} -> [{port,11210}, {dbdir,"/var/membase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine,"./bin/bucket_engine/bucket_engine.so"}, {engines,[{membase,[{engine,"bin/ep_engine/ep.so"}, {initfile,"/etc/membase/priv/init.sql"}]}, {memcached,[{engine,"lib/memcached/default_engine.so"}]}]}, {verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',moxi} -> [{port,11211},{verbosity,[]}] INFO REPORT <0.118.0> 2011-04-08 16:04:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',ns_log} -> [{filename,"/var/membase/data/ns_log"}] INFO REPORT <0.118.0> 2011-04-08 16:04:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',rest} -> [{port,8091}] INFO REPORT <0.118.0> 2011-04-08 16:04:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:00 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:00 =============================================================================== Pushing config done INFO REPORT <0.431.0> 2011-04-08 16:04:04 =============================================================================== Detected a new node (from node 'ns_1@10.1.5.193'). Moving config around. INFO REPORT <0.431.0> 2011-04-08 16:04:06 =============================================================================== ns_node_disco_log: nodes changed: ['ns_1@10.1.5.178','ns_1@10.1.5.192', 'ns_1@10.1.5.193'] INFO REPORT <0.439.0> 2011-04-08 16:04:06 =============================================================================== Pulling config INFO REPORT <0.439.0> 2011-04-08 16:04:06 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.439.0> 2011-04-08 16:04:06 =============================================================================== Pulling config done INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> active INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== config change: rebalance_status -> running INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config done INFO REPORT <0.476.0> 2011-04-08 16:04:14 =============================================================================== ns_1@10.1.5.193:<0.476.0>:ns_bucket_sup:79: Starting new child: {{per_bucket_sup, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {single_bucket_sup, start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, permanent, infinity, supervisor, [single_bucket_sup]} INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}]}]}] PROGRESS REPORT <0.477.0> 2011-04-08 16:04:14 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<0.557.0>}, {name,{per_bucket_sup,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{single_bucket_sup,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:14 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:14 =============================================================================== Pushing config done INFO REPORT <0.559.0> 2011-04-08 16:04:15 =============================================================================== ns_1@10.1.5.193:<0.559.0>:ns_memcached:438: Created bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" with config string "vb0=false;waitforwarmup=false;ht_size=3079;ht_locks=5;failpartialwarmup=false;db_shards=4;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB;tap_noop_interval=20;max_txn_size=1000;max_size=419430400;initfile=/etc/membase/priv/init.sql;tap_keepalive=300;dbname=/var/membase/data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561;" INFO REPORT <0.559.0> 2011-04-08 16:04:16 =============================================================================== ns_log: logging ns_memcached:1:Bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" loaded on node 'ns_1@10.1.5.193' in 2 seconds. PROGRESS REPORT <0.558.0> 2011-04-08 16:04:16 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.559.0>}, {name,{ns_memcached,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{ns_memcached,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] PROGRESS REPORT <0.558.0> 2011-04-08 16:04:16 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.568.0>}, {name,{ns_vbm_sup,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{ns_vbm_sup,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] PROGRESS REPORT <0.558.0> 2011-04-08 16:04:16 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.569.0>}, {name,{stats_collector,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{stats_collector,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.558.0> 2011-04-08 16:04:16 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.570.0>}, {name,{stats_archiver,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{stats_archiver,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute' PROGRESS REPORT <0.558.0> 2011-04-08 16:04:16 =============================================================================== supervisor {local,'single_bucket_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.571.0>}, {name,{stats_reader,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{stats_reader,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,541725},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,935,<0.573.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,541725},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,935,<0.573.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour' INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,566829},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,936,<0.580.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,566829},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,936,<0.580.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day' INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,584742},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,937,<0.587.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,584742},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,937,<0.587.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week' INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,590179},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,938,<0.594.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,590179},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,938,<0.594.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month' INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,622950},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,939,<0.601.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,622950},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,939,<0.601.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:108: Creating table 'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year' INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,637785},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,940,<0.608.0>}} INFO REPORT <0.67.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.67.0>:mb_mnesia:199: Mnesia table event: {write,{schema,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year', [{name,'stats_archiver-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.5.193']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1302,303856,637785},'ns_1@10.1.5.193'}}, {version,{{2,0},[]}}]}, {tid,940,<0.608.0>}} INFO REPORT <0.569.0> 2011-04-08 16:04:16 =============================================================================== ns_1@10.1.5.193:<0.569.0>:stats_collector:80: Stats for bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561": auth_cmds 0 auth_errors 0 bucket_conns 1 bytes_read 72 bytes_written 15648 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 0 conn_yields 0 connection_structures 11 curr_connections 11 curr_items 0 curr_items_tot 0 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 0 ep_commit_time 0 ep_commit_time_total 0 ep_data_age 0 ep_data_age_highwat 0 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /var/membase/data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561 ep_dbshards 4 ep_diskqueue_drain 0 ep_diskqueue_fill 0 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_duration 0 ep_flush_duration_highwat 0 ep_flush_duration_total 0 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 0 ep_io_read_bytes 0 ep_io_write_bytes 0 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 0 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 0 ep_max_data_size 419430400 ep_max_txn_size 1000 ep_mem_high_wat 314572800 ep_mem_low_wat 251658240 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs0 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 600 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 0 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 0 ep_total_del_items 0 ep_total_enqueued 0 ep_total_new_items 0 ep_total_persisted 0 ep_vb_total 0 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.6.5.3_143_g366c22e ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 14934 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.7-rc limit_maxbytes 67108864 mem_used 600 pid 20974 pointer_size 64 proxy_cmd_count 0 proxy_cmd_time 0 proxy_local_cmd_count 0 proxy_local_cmd_time 0 rejected_conns 0 rusage_system 0.088986 rusage_user 0.514921 threads 4 time 1302303855 total_connections 11 uptime 23 vb_active_curr_items 0 vb_active_eject 0 vb_active_ht_memory 0 vb_active_itm_memory 0 vb_active_num 0 vb_active_num_non_resident 0 vb_active_ops_create 0 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 0 vb_active_queue_age 0 vb_active_queue_drain 0 vb_active_queue_fill 0 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 0 vb_replica_eject 0 vb_replica_ht_memory 0 vb_replica_itm_memory 0 vb_replica_num 0 vb_replica_num_non_resident 0 vb_replica_ops_create 0 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 0 vb_replica_queue_age 0 vb_replica_queue_drain 0 vb_replica_queue_fill 0 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_430_g6235145 INFO REPORT <0.439.0> 2011-04-08 16:04:17 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.118.0> 2011-04-08 16:04:17 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}, {fastForwardMap,[['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178'|...], [...]|...]}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:04:17 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:17 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:17 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:04:17 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:04:17 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:04:17 =============================================================================== Pushing config done INFO REPORT <0.460.0> 2011-04-08 16:04:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303887,657358}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,17589288}, {processes,7324480}, {processes_used,7314840}, {system,10264808}, {atom,519117}, {atom_used,500042}, {binary,237168}, {code,4055640}, {ets,543684}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,357}, {memory_data,{2075373568,1706618880,{<10509.124.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 312920 kB\nBuffers: 246336 kB\nCached: 1250532 kB\nSwapCached: 0 kB\nActive: 728748 kB\nInactive: 886496 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 311284 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 6020 kB\nWriteback: 0 kB\nAnonPages: 118392 kB\nMapped: 29168 kB\nSlab: 85292 kB\nPageTables: 2276 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 515536 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1280544768}, {buffered_memory,252248064}, {free_memory,320430080}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{351918,2}}, {context_switches,{44730,0}}, {garbage_collection,{13676,55049752,0}}, {io,{{input,5174939},{output,1694485}}}, {reductions,{14395719,5081118}}, {run_queue,0}, {runtime,{3330,1480}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,303887,738593}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34542672}, {processes,14104792}, {processes_used,14087544}, {system,20437880}, {atom,824289}, {atom_used,817419}, {binary,178216}, {code,8617057}, {ets,1404424}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,436}, {memory_data,{2058928128,1576886272,{<10461.124.0>,1572784}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 470372 kB\nBuffers: 352548 kB\nCached: 895060 kB\nSwapCached: 17552 kB\nActive: 760956 kB\nInactive: 629724 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 470372 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 3040 kB\nWriteback: 0 kB\nAnonPages: 138792 kB\nMapped: 17604 kB\nSlab: 124564 kB\nPageTables: 6508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 584636 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,916541440}, {buffered_memory,361009152}, {free_memory,481660928}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{431236,14}}, {context_switches,{88417,0}}, {garbage_collection,{22440,82388991,0}}, {io,{{input,6299192},{output,4240197}}}, {reductions,{28050155,2183648}}, {run_queue,0}, {runtime,{4100,460}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,303887,894958}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,32724416}, {processes,13339360}, {processes_used,13322752}, {system,19385056}, {atom,784585}, {atom_used,765250}, {binary,280600}, {code,7930444}, {ets,1084096}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,404}, {memory_data,{2058928128,1000730624,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 995728 kB\nBuffers: 252624 kB\nCached: 506652 kB\nSwapCached: 17896 kB\nActive: 651812 kB\nInactive: 228448 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 995728 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 6356 kB\nWriteback: 0 kB\nAnonPages: 116400 kB\nMapped: 20084 kB\nSlab: 111208 kB\nPageTables: 5892 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 529040 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,518811648}, {buffered_memory,258686976}, {free_memory,1019625472}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{394806,2}}, {context_switches,{43372,0}}, {garbage_collection,{9191,20050766,0}}, {io,{{input,5343709},{output,1872213}}}, {reductions,{7302852,630348}}, {run_queue,0}, {runtime,{1510,130}}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:05:00 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:05:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:05:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:05:00 =============================================================================== config change: rebalance_status -> none INFO REPORT <0.118.0> 2011-04-08 16:05:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:05:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:05:00 =============================================================================== Pushing config done INFO REPORT <0.439.0> 2011-04-08 16:05:01 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' PROGRESS REPORT <0.568.0> 2011-04-08 16:05:04 =============================================================================== supervisor {local,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.668.0>}, {name, {child_id, [853,852,851,850,849,848,847,846,845,844,843,842,841,840, 839,838,837,836,835,834,833,832,831,830,829,828,827,826, 825,824,823,822,821,820,819,818,817,816,815,814,813,812, 811,810,809,808,807,806,805,804,803,802,801,800,799,798, 797,796,795,794,793,792,791,790,789,788,787,786,785,784, 783,782,781,780,779,778,777,776,775,774,773,772,771,770, 769,768,767,766,765,764,763,762,761,760,759,758,757,756, 755,754,753,752,751,750,749,748,747,746,745,744,743,742, 741,740,739,738,737,736,735,734,733,732,731,730,729,728, 727,726,725,724,723,722,721,720,719,718,717,716,715,714, 713,712,711,710,709,708,707,706,705,704,703,702,701,700, 699,698,697,696,695,694,693,692,691,690,689,688,687,686, 685,684,683], 'ns_1@10.1.5.178'}}, {mfa, {ebucketmigrator_srv,start_link, [{"10.1.5.193",11210}, {"10.1.5.178",11210}, [{username, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {password,[]}, {vbuckets, [853,852,851,850,849,848,847,846,845,844,843,842,841, 840,839,838,837,836,835,834,833,832,831,830,829,828, 827,826,825,824,823,822,821,820,819,818,817,816,815, 814,813,812,811,810,809,808,807,806,805,804,803,802, 801,800,799,798,797,796,795,794,793,792,791,790,789, 788,787,786,785,784,783,782,781,780,779,778,777,776, 775,774,773,772,771,770,769,768,767,766,765,764,763, 762,761,760,759,758,757,756,755,754,753,752,751,750, 749,748,747,746,745,744,743,742,741,740,739,738,737, 736,735,734,733,732,731,730,729,728,727,726,725,724, 723,722,721,720,719,718,717,716,715,714,713,712,711, 710,709,708,707,706,705,704,703,702,701,700,699,698, 697,696,695,694,693,692,691,690,689,688,687,686,685, 684,683]}, {takeover,false}, {suffix, "ns_1@10.1.5.178-ns_1@10.1.5.192-1302303904.105684"}]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] PROGRESS REPORT <0.568.0> 2011-04-08 16:05:04 =============================================================================== supervisor {local,'ns_vbm_sup-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561'} started [{pid,<0.669.0>}, {name, {child_id, [1023,1022,1021,1020,1019,1018,1017,1016,1015,1014,1013, 1012,1011,1010,1009,1008,1007,1006,1005,1004,1003,1002, 1001,1000,999,998,997,996,995,994,993,992,991,990,989,988, 987,986,985,984,983,982,981,980,979,978,977,976,975,974, 973,972,971,970,969,968,967,966,965,964,963,962,961,960, 959,958,957,956,955,954,953,952,951,950,949,948,947,946, 945,944,943,942,941,940,939,938,937,936,935,934,933,932, 931,930,929,928,927,926,925,924,923,922,921,920,919,918, 917,916,915,914,913,912,911,910,909,908,907,906,905,904, 903,902,901,900,899,898,897,896,895,894,893,892,891,890, 889,888,887,886,885,884,883,882,881,880,879,878,877,876, 875,874,873,872,871,870,869,868,867,866,865,864,863,862, 861,860,859,858,857,856,855,854], 'ns_1@10.1.5.192'}}, {mfa, {ebucketmigrator_srv,start_link, [{"10.1.5.193",11210}, {"10.1.5.192",11210}, [{username, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {password,[]}, {vbuckets, [1023,1022,1021,1020,1019,1018,1017,1016,1015,1014, 1013,1012,1011,1010,1009,1008,1007,1006,1005,1004, 1003,1002,1001,1000,999,998,997,996,995,994,993,992, 991,990,989,988,987,986,985,984,983,982,981,980,979, 978,977,976,975,974,973,972,971,970,969,968,967,966, 965,964,963,962,961,960,959,958,957,956,955,954,953, 952,951,950,949,948,947,946,945,944,943,942,941,940, 939,938,937,936,935,934,933,932,931,930,929,928,927, 926,925,924,923,922,921,920,919,918,917,916,915,914, 913,912,911,910,909,908,907,906,905,904,903,902,901, 900,899,898,897,896,895,894,893,892,891,890,889,888, 887,886,885,884,883,882,881,880,879,878,877,876,875, 874,873,872,871,870,869,868,867,866,865,864,863,862, 861,860,859,858,857,856,855,854]}, {takeover,false}, {suffix, "ns_1@10.1.5.192-ns_1@10.1.5.192-1302303904.191769"}]]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] ERROR REPORT <0.569.0> 2011-04-08 16:05:18 =============================================================================== ns_1@10.1.5.193:<0.569.0>:stats_collector:121: Dropped 1 ticks ERROR REPORT <0.479.0> 2011-04-08 16:05:19 =============================================================================== ns_1@10.1.5.193:<0.479.0>:system_stats_collector:128: lost 1 ticks INFO REPORT <0.439.0> 2011-04-08 16:05:24 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.460.0> 2011-04-08 16:05:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,303947,657281}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,18762920}, {processes,8172404}, {processes_used,8163308}, {system,10590516}, {atom,519117}, {atom_used,500441}, {binary,395896}, {code,4062708}, {ets,699396}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,417}, {memory_data,{2075373568,1756721152,{<10509.512.0>,3814172}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 303240 kB\nBuffers: 246336 kB\nCached: 1251668 kB\nSwapCached: 0 kB\nActive: 738060 kB\nInactive: 887348 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 301604 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 428 kB\nWriteback: 0 kB\nAnonPages: 127452 kB\nMapped: 29228 kB\nSlab: 85296 kB\nPageTables: 2284 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 525940 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1281708032}, {buffered_memory,252248064}, {free_memory,310517760}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{412265,350}}, {context_switches,{71556,0}}, {garbage_collection,{32997,232203089,0}}, {io,{{input,6245948},{output,3455579}}}, {reductions,{51811147,6353560}}, {run_queue,0}, {runtime,{13760,1770}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,303947,736723}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,37105632}, {processes,16339584}, {processes_used,16321120}, {system,20766048}, {atom,824289}, {atom_used,817742}, {binary,321344}, {code,8617057}, {ets,1564288}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,496}, {memory_data,{2058928128,1585283072,{<10461.1839.0>,3144104}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 463536 kB\nBuffers: 352548 kB\nCached: 899280 kB\nSwapCached: 17552 kB\nActive: 767344 kB\nInactive: 630272 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 463536 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 304 kB\nWriteback: 0 kB\nAnonPages: 141516 kB\nMapped: 17612 kB\nSlab: 124744 kB\nPageTables: 6528 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 586836 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,920862720}, {buffered_memory,361009152}, {free_memory,474660864}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{491245,23}}, {context_switches,{123853,0}}, {garbage_collection,{32777,132053381,0}}, {io,{{input,9170115},{output,6487868}}}, {reductions,{48259631,3944180}}, {run_queue,0}, {runtime,{6530,350}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,303946,912476}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,37231104}, {processes,17528584}, {processes_used,17513992}, {system,19702520}, {atom,784585}, {atom_used,765809}, {binary,297344}, {code,7944128}, {ets,1365176}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,463}, {memory_data,{2058928128,1065332736,{<0.569.0>,2543288}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 965084 kB\nBuffers: 252624 kB\nCached: 509128 kB\nSwapCached: 17896 kB\nActive: 681556 kB\nInactive: 228724 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 965084 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 284 kB\nWriteback: 0 kB\nAnonPages: 143940 kB\nMapped: 20300 kB\nSlab: 111496 kB\nPageTables: 5948 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 560200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,521347072}, {buffered_memory,258686976}, {free_memory,988246016}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{454989,184}}, {context_switches,{68067,0}}, {garbage_collection,{25547,159673839,0}}, {io,{{input,5943309},{output,3169224}}}, {reductions,{37037919,6049279}}, {run_queue,0}, {runtime,{10760,1910}}]}]}] INFO REPORT <0.569.0> 2011-04-08 16:05:57 =============================================================================== ns_1@10.1.5.193:<0.569.0>:stats_collector:80: Stats for bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561": auth_cmds 346 auth_errors 0 bucket_conns 6 bytes_read 980274 bytes_written 31539333 cas_badval 0 cas_hits 0 cas_misses 0 cmd_flush 0 cmd_get 0 cmd_set 31 conn_yields 12 connection_structures 16 curr_connections 16 curr_items 45 curr_items_tot 86 daemon_connections 10 decr_hits 0 decr_misses 0 delete_hits 0 delete_misses 0 ep_bg_fetched 0 ep_commit_num 64 ep_commit_time 0 ep_commit_time_total 8 ep_data_age 0 ep_data_age_highwat 2 ep_db_cleaner_status complete ep_db_strategy multiMTVBDB ep_dbinit 1 ep_dbname /var/membase/data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561-data/FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561 ep_dbshards 4 ep_diskqueue_drain 86 ep_diskqueue_fill 86 ep_diskqueue_items 0 ep_diskqueue_memory 0 ep_diskqueue_pending 0 ep_expired 0 ep_flush_duration 0 ep_flush_duration_highwat 2 ep_flush_duration_total 8 ep_flush_preempts 0 ep_flusher_state running ep_flusher_todo 0 ep_io_num_read 0 ep_io_num_write 86 ep_io_read_bytes 0 ep_io_write_bytes 884326 ep_item_begin_failed 0 ep_item_commit_failed 0 ep_item_flush_expired 0 ep_item_flush_failed 0 ep_items_rm_from_checkpoints 0 ep_kv_size 893620 ep_latency_arith_cmd 0 ep_latency_get_cmd 0 ep_latency_store_cmd 31 ep_max_data_size 419430400 ep_max_txn_size 1000 ep_mem_high_wat 314572800 ep_mem_low_wat 251658240 ep_min_data_age 0 ep_num_active_non_resident 0 ep_num_checkpoint_remover_runs20 ep_num_eject_failures 0 ep_num_eject_replicas 0 ep_num_expiry_pager_runs 0 ep_num_non_resident 0 ep_num_not_my_vbuckets 0 ep_num_pager_runs 0 ep_num_value_ejects 0 ep_onlineupdate false ep_onlineupdate_revert_add 0 ep_onlineupdate_revert_delete 0 ep_onlineupdate_revert_update 0 ep_oom_errors 0 ep_overhead 17721342 ep_pending_ops 0 ep_pending_ops_max 0 ep_pending_ops_max_duration 0 ep_pending_ops_total 0 ep_queue_age_cap 900 ep_queue_size 0 ep_storage_age 0 ep_storage_age_highwat 2 ep_storage_type featured ep_store_max_concurrency 10 ep_store_max_readers 9 ep_store_max_readwrite 1 ep_tap_bg_fetch_requeued 0 ep_tap_bg_fetched 0 ep_tap_keepalive 300 ep_tmp_oom_errors 0 ep_too_old 0 ep_too_young 0 ep_total_cache_size 892938 ep_total_del_items 0 ep_total_enqueued 86 ep_total_new_items 86 ep_total_persisted 86 ep_vb_total 682 ep_vbucket_del 0 ep_vbucket_del_fail 0 ep_version 1.6.5.3_143_g366c22e ep_warmed_up 0 ep_warmup true ep_warmup_dups 0 ep_warmup_oom 0 ep_warmup_thread complete ep_warmup_time 14934 get_hits 0 get_misses 0 incr_hits 0 incr_misses 0 libevent 2.0.7-rc limit_maxbytes 67108864 mem_used 18614962 pid 20974 pointer_size 64 proxy_cmd_count 0 proxy_cmd_time 0 proxy_local_cmd_count 0 proxy_local_cmd_time 0 rejected_conns 0 rusage_system 0.705892 rusage_user 3.576456 tap_connect_received 2 tap_mutation_received 55 tap_mutation_sent 45 tap_opaque_received 684 tap_opaque_sent 343 tap_vbucket_set_received 682 threads 4 time 1302303956 total_connections 357 uptime 124 vb_active_curr_items 45 vb_active_eject 0 vb_active_ht_memory 8535912 vb_active_itm_memory 467235 vb_active_num 341 vb_active_num_non_resident 0 vb_active_ops_create 45 vb_active_ops_delete 0 vb_active_ops_reject 0 vb_active_ops_update 0 vb_active_perc_mem_resident 100 vb_active_queue_age 0 vb_active_queue_drain 45 vb_active_queue_fill 45 vb_active_queue_memory 0 vb_active_queue_pending 0 vb_active_queue_size 0 vb_dead_num 0 vb_pending_curr_items 0 vb_pending_eject 0 vb_pending_ht_memory 0 vb_pending_itm_memory 0 vb_pending_num 0 vb_pending_num_non_resident 0 vb_pending_ops_create 0 vb_pending_ops_delete 0 vb_pending_ops_reject 0 vb_pending_ops_update 0 vb_pending_perc_mem_resident 0 vb_pending_queue_age 0 vb_pending_queue_drain 0 vb_pending_queue_fill 0 vb_pending_queue_memory 0 vb_pending_queue_pending 0 vb_pending_queue_size 0 vb_replica_curr_items 41 vb_replica_eject 0 vb_replica_ht_memory 8535912 vb_replica_itm_memory 425703 vb_replica_num 341 vb_replica_num_non_resident 0 vb_replica_ops_create 41 vb_replica_ops_delete 0 vb_replica_ops_reject 0 vb_replica_ops_update 0 vb_replica_perc_mem_resident 100 vb_replica_queue_age 0 vb_replica_queue_drain 41 vb_replica_queue_fill 41 vb_replica_queue_memory 0 vb_replica_queue_pending 0 vb_replica_queue_size 0 version 1.4.4_430_g6235145 INFO REPORT <0.439.0> 2011-04-08 16:06:05 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.439.0> 2011-04-08 16:06:18 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.439.0> 2011-04-08 16:06:32 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.460.0> 2011-04-08 16:06:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304007,656358}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,18693360}, {processes,7867788}, {processes_used,7858692}, {system,10825572}, {atom,519117}, {atom_used,500441}, {binary,541720}, {code,4062708}, {ets,787796}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,477}, {memory_data,{2075373568,1766633472,{<10509.512.0>,1457156}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 299520 kB\nBuffers: 246336 kB\nCached: 1252916 kB\nSwapCached: 0 kB\nActive: 740364 kB\nInactive: 888284 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 297884 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 112 kB\nWriteback: 0 kB\nAnonPages: 129416 kB\nMapped: 29228 kB\nSlab: 85304 kB\nPageTables: 2288 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 526124 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282985984}, {buffered_memory,252248064}, {free_memory,306708480}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{472268,352}}, {context_switches,{98168,0}}, {garbage_collection,{65117,421602982,0}}, {io,{{input,6534649},{output,4426957}}}, {reductions,{92371652,7039826}}, {run_queue,0}, {runtime,{24800,1900}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304007,729735}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,36560184}, {processes,15689712}, {processes_used,15669648}, {system,20870472}, {atom,824289}, {atom_used,817742}, {binary,266712}, {code,8617057}, {ets,1719632}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,556}, {memory_data,{2058928128,1584140288,{<10461.1839.0>,3145928}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 465520 kB\nBuffers: 352552 kB\nCached: 898524 kB\nSwapCached: 17552 kB\nActive: 765796 kB\nInactive: 629668 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 465520 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 312 kB\nWriteback: 0 kB\nAnonPages: 140124 kB\nMapped: 17612 kB\nSlab: 124564 kB\nPageTables: 6516 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 574876 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,920088576}, {buffered_memory,361013248}, {free_memory,476692480}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{551242,20}}, {context_switches,{144554,0}}, {garbage_collection,{38964,170090567,0}}, {io,{{input,15943891},{output,8449583}}}, {reductions,{71215824,3373701}}, {run_queue,0}, {runtime,{8680,280}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304006,911041}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,36980808}, {processes,17068928}, {processes_used,17054336}, {system,19911880}, {atom,784585}, {atom_used,765809}, {binary,345952}, {code,7944128}, {ets,1524472}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,523}, {memory_data,{2058928128,1072967680,{<0.559.0>,2914272}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 963224 kB\nBuffers: 252624 kB\nCached: 510504 kB\nSwapCached: 17896 kB\nActive: 682820 kB\nInactive: 229268 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 963224 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 176 kB\nWriteback: 0 kB\nAnonPages: 144404 kB\nMapped: 20300 kB\nSlab: 111512 kB\nPageTables: 5952 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 559380 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,522756096}, {buffered_memory,258686976}, {free_memory,986341376}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{514972,166}}, {context_switches,{94119,0}}, {garbage_collection,{51700,342171258,0}}, {io,{{input,6246858},{output,4161183}}}, {reductions,{75767855,6737234}}, {run_queue,0}, {runtime,{22370,1950}}]}]}] INFO REPORT <0.476.0> 2011-04-08 16:06:59 =============================================================================== ns_1@10.1.5.193:<0.476.0>:ns_bucket_sup:86: Stopping child for dead bucket: {{per_bucket_sup, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, <0.557.0>, supervisor, [single_bucket_sup]} INFO REPORT <0.557.0> 2011-04-08 16:06:59 =============================================================================== ns_1@10.1.5.193:<0.557.0>:single_bucket_sup:27: Delegating exit {'EXIT', <0.477.0>, shutdown} to child supervisor: <0.558.0> INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178','ns_1@10.1.5.192'], ['ns_1@10.1.5.178'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.559.0> 2011-04-08 16:06:59 =============================================================================== ns_log: logging ns_memcached:2:Shutting down bucket "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" on 'ns_1@10.1.5.193' for server shutdown INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config INFO REPORT <0.557.0> 2011-04-08 16:06:59 =============================================================================== ns_1@10.1.5.193:<0.557.0>:single_bucket_sup:24: per-bucket supervisor for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" died with reason shutdown INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== config change: {node,'ns_1@10.1.5.193',membership} -> inactiveFailed INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:06:59 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:06:59 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:00 =============================================================================== config change: {node,'ns_1@10.1.5.178',membership} -> inactiveFailed INFO REPORT <0.118.0> 2011-04-08 16:07:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:07:00 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:00 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:00 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:07:00 =============================================================================== Pushing config done INFO REPORT <0.439.0> 2011-04-08 16:07:02 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.118.0> 2011-04-08 16:07:15 =============================================================================== config change: rebalance_status -> running INFO REPORT <0.118.0> 2011-04-08 16:07:15 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:15 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config done INFO REPORT <0.476.0> 2011-04-08 16:07:15 =============================================================================== ns_1@10.1.5.193:<0.476.0>:ns_bucket_sup:79: Starting new child: {{per_bucket_sup, "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}, {single_bucket_sup, start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, permanent, infinity, supervisor, [single_bucket_sup]} PROGRESS REPORT <0.477.0> 2011-04-08 16:07:15 =============================================================================== supervisor {local,ns_bucket_sup} started [{pid,<0.816.0>}, {name,{per_bucket_sup,"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"}}, {mfa,{single_bucket_sup,start_link, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] INFO REPORT <0.118.0> 2011-04-08 16:07:15 =============================================================================== config change: buckets -> [{configs,[{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561", [{num_replicas,1}, {ram_quota,419430400}, {auth_type,none}, {moxi_port,11220}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.5.192','ns_1@10.1.5.193','ns_1@10.1.5.178']}, {map,[['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192',undefined], ['ns_1@10.1.5.192'|...], [...]|...]}, {fastForwardMap,undefined}]}]}] INFO REPORT <0.118.0> 2011-04-08 16:07:15 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:15 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config INFO REPORT <0.818.0> 2011-04-08 16:07:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:07:15 =============================================================================== Pushing config done INFO REPORT <0.818.0> 2011-04-08 16:07:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.466.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.828.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.828.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.830.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.118.0> 2011-04-08 16:07:26 =============================================================================== config change: rebalance_status -> {none,<<"Rebalance failed. See logs for detailed reason. You can try rebalance again.">>} INFO REPORT <0.118.0> 2011-04-08 16:07:26 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:26 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:07:26 =============================================================================== Pushing config done INFO REPORT <0.118.0> 2011-04-08 16:07:26 =============================================================================== ns_node_disco_conf_events config all INFO REPORT <0.439.0> 2011-04-08 16:07:26 =============================================================================== Pushing config INFO REPORT <0.439.0> 2011-04-08 16:07:26 =============================================================================== Pushing config done INFO REPORT <0.818.0> 2011-04-08 16:07:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.830.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.840.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.840.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.842.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.842.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.845.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:07:38 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:07:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.845.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.847.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:07:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304067,655772}}, {active_buckets,[]}, {memory, [{total,15287792}, {processes,4773684}, {processes_used,4761804}, {system,10514108}, {atom,519117}, {atom_used,500490}, {binary,205736}, {code,4062708}, {ets,812820}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,537}, {memory_data,{2075373568,1768157184,{<10509.512.0>,1457156}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 301480 kB\nBuffers: 246336 kB\nCached: 1253236 kB\nSwapCached: 0 kB\nActive: 738164 kB\nInactive: 888452 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 299844 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 364 kB\nWriteback: 0 kB\nAnonPages: 127040 kB\nMapped: 28212 kB\nSlab: 85392 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 517908 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1283313664}, {buffered_memory,252248064}, {free_memory,308715520}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{531917,1}}, {context_switches,{110839,0}}, {garbage_collection,{77407,483707092,0}}, {io,{{input,6956265},{output,5104609}}}, {reductions,{105658355,126108}}, {run_queue,0}, {runtime,{28320,10}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304067,729405}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35237704}, {processes,14300832}, {processes_used,14279592}, {system,20936872}, {atom,828329}, {atom_used,823498}, {binary,184752}, {code,8669006}, {ets,1809560}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,616}, {memory_data,{2058928128,1582489600,{<10461.2477.0>,2543360}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 464644 kB\nBuffers: 352552 kB\nCached: 899548 kB\nSwapCached: 17552 kB\nActive: 766152 kB\nInactive: 630244 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 464644 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 352 kB\nWriteback: 0 kB\nAnonPages: 140040 kB\nMapped: 17612 kB\nSlab: 124640 kB\nPageTables: 6508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 576220 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,921137152}, {buffered_memory,361013248}, {free_memory,475795456}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{611237,15}}, {context_switches,{164203,0}}, {garbage_collection,{44562,197823623,0}}, {io,{{input,16437346},{output,10082599}}}, {reductions,{86580851,1144580}}, {run_queue,0}, {runtime,{10230,170}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304066,894399}}, {active_buckets,[]}, {memory, [{total,30164576}, {processes,10338624}, {processes_used,10317152}, {system,19825952}, {atom,784585}, {atom_used,765874}, {binary,202720}, {code,7944128}, {ets,1580248}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,583}, {memory_data,{2058928128,1068777472,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 965804 kB\nBuffers: 252628 kB\nCached: 510844 kB\nSwapCached: 17896 kB\nActive: 679468 kB\nInactive: 229788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 965804 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 372 kB\nWriteback: 0 kB\nAnonPages: 141224 kB\nMapped: 21920 kB\nSlab: 111624 kB\nPageTables: 6248 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 555504 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523104256}, {buffered_memory,258691072}, {free_memory,988983296}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{574806,2}}, {context_switches,{108563,0}}, {garbage_collection,{62083,414105582,0}}, {io,{{input,8820337},{output,6940421}}}, {reductions,{91117215,123554}}, {run_queue,0}, {runtime,{26900,30}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.847.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.851.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.851.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.853.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:07:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.853.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:07:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.856.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:07:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:07:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:08:00 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:08:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.856.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.858.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.858.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.861.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.861.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.863.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.863.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.866.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.866.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.868.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.868.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.871.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.871.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.873.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.873.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.876.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.876.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.878.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:08:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304127,655884}}, {active_buckets,[]}, {memory, [{total,15311824}, {processes,4767852}, {processes_used,4755972}, {system,10543972}, {atom,519117}, {atom_used,500490}, {binary,216216}, {code,4062708}, {ets,830484}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,597}, {memory_data,{2075373568,1766633472,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 294180 kB\nBuffers: 246344 kB\nCached: 1251960 kB\nSwapCached: 0 kB\nActive: 746804 kB\nInactive: 887528 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 292544 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 744 kB\nWriteback: 0 kB\nAnonPages: 136032 kB\nMapped: 32228 kB\nSlab: 85424 kB\nPageTables: 2328 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 559300 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282007040}, {buffered_memory,252256256}, {free_memory,301240320}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{591916,1}}, {context_switches,{116377,0}}, {garbage_collection,{78465,486779782,0}}, {io,{{input,9024820},{output,5933618}}}, {reductions,{106494435,143337}}, {run_queue,0}, {runtime,{28470,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304127,729926}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,36958408}, {processes,15891720}, {processes_used,15870480}, {system,21066688}, {atom,828329}, {atom_used,823498}, {binary,159992}, {code,8669006}, {ets,1962680}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,676}, {memory_data,{2058928128,1584140288,{<10461.2477.0>,2543552}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 463784 kB\nBuffers: 352552 kB\nCached: 900340 kB\nSwapCached: 17552 kB\nActive: 766876 kB\nInactive: 630792 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 463784 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 192 kB\nWriteback: 0 kB\nAnonPages: 140524 kB\nMapped: 17612 kB\nSlab: 124576 kB\nPageTables: 6508 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571868 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,921948160}, {buffered_memory,361013248}, {free_memory,474914816}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{671236,14}}, {context_switches,{178113,0}}, {garbage_collection,{49170,223514955,0}}, {io,{{input,16736153},{output,10981216}}}, {reductions,{93599642,1202413}}, {run_queue,0}, {runtime,{11280,170}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304126,894411}}, {active_buckets,[]}, {memory, [{total,29674576}, {processes,9875944}, {processes_used,9854472}, {system,19798632}, {atom,784585}, {atom_used,765874}, {binary,207112}, {code,7944128}, {ets,1546888}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,643}, {memory_data,{2058928128,1068896256,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966828 kB\nBuffers: 252628 kB\nCached: 510952 kB\nSwapCached: 17896 kB\nActive: 678372 kB\nInactive: 229784 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966828 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 136 kB\nWriteback: 0 kB\nAnonPages: 140016 kB\nMapped: 20308 kB\nSlab: 111520 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523214848}, {buffered_memory,258691072}, {free_memory,990031872}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{634806,2}}, {context_switches,{113421,0}}, {garbage_collection,{62720,416543702,0}}, {io,{{input,9077475},{output,7097841}}}, {reductions,{91865716,117232}}, {run_queue,0}, {runtime,{27050,20}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.878.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.882.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.882.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.884.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:08:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.884.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:08:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.887.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:08:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:08:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:08:59 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:09:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.887.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.889.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.889.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.892.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.892.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.894.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:09:16 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:09:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.894.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.897.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.897.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.899.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.899.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.902.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.902.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.904.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.904.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.907.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.907.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.909.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:09:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304187,652057}}, {active_buckets,[]}, {memory, [{total,15664664}, {processes,5143660}, {processes_used,5131780}, {system,10521004}, {atom,519117}, {atom_used,500490}, {binary,208656}, {code,4062708}, {ets,814140}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,657}, {memory_data,{2075373568,1765498880,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302620 kB\nBuffers: 246344 kB\nCached: 1252064 kB\nSwapCached: 0 kB\nActive: 737892 kB\nInactive: 887576 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300984 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 68 kB\nWriteback: 0 kB\nAnonPages: 127072 kB\nMapped: 28212 kB\nSlab: 85336 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282113536}, {buffered_memory,252256256}, {free_memory,309882880}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{651915,1}}, {context_switches,{121273,0}}, {garbage_collection,{79369,489298986,0}}, {io,{{input,9281952},{output,6090850}}}, {reductions,{107251389,128436}}, {run_queue,0}, {runtime,{28560,10}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304187,729681}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34704984}, {processes,13456288}, {processes_used,13434968}, {system,21248696}, {atom,828329}, {atom_used,823498}, {binary,177072}, {code,8669006}, {ets,2124424}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,736}, {memory_data,{2058928128,1582743552,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 464528 kB\nBuffers: 352552 kB\nCached: 900680 kB\nSwapCached: 17552 kB\nActive: 765192 kB\nInactive: 631412 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 464528 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 364 kB\nWriteback: 0 kB\nAnonPages: 139136 kB\nMapped: 17612 kB\nSlab: 124668 kB\nPageTables: 6504 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 570916 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,922296320}, {buffered_memory,361013248}, {free_memory,475676672}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{731237,14}}, {context_switches,{190472,0}}, {garbage_collection,{53537,243406701,0}}, {io,{{input,19449008},{output,14027335}}}, {reductions,{100066229,1131146}}, {run_queue,0}, {runtime,{12290,220}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304187,894953}}, {active_buckets,[]}, {memory, [{total,29755112}, {processes,9924032}, {processes_used,9902560}, {system,19831080}, {atom,784585}, {atom_used,765874}, {binary,206056}, {code,7944128}, {ets,1578936}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,704}, {memory_data,{2058928128,1068896256,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966828 kB\nBuffers: 252628 kB\nCached: 511072 kB\nSwapCached: 17896 kB\nActive: 678376 kB\nInactive: 229788 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966828 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 176 kB\nWriteback: 0 kB\nAnonPages: 139908 kB\nMapped: 20308 kB\nSlab: 111520 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523337728}, {buffered_memory,258691072}, {free_memory,990031872}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{694806,2}}, {context_switches,{118288,0}}, {garbage_collection,{63403,419100257,0}}, {io,{{input,9333802},{output,7257581}}}, {reductions,{92641086,147430}}, {run_queue,0}, {runtime,{27190,20}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.909.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.913.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.913.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.915.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:09:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.915.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:09:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.918.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:09:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:09:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.918.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.920.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:10:07 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:10:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.920.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.938.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.938.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.940.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.940.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.943.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.943.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.945.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:10:26 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:10:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.945.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.948.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:10:32 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:10:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.948.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.950.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.950.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.953.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.953.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.955.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:10:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304247,652947}}, {active_buckets,[]}, {memory, [{total,15346872}, {processes,4815012}, {processes_used,4803132}, {system,10531860}, {atom,519117}, {atom_used,500490}, {binary,201192}, {code,4062708}, {ets,831628}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,717}, {memory_data,{2075373568,1765490688,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302496 kB\nBuffers: 246344 kB\nCached: 1252184 kB\nSwapCached: 0 kB\nActive: 737932 kB\nInactive: 887632 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300860 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 212 kB\nWriteback: 0 kB\nAnonPages: 127072 kB\nMapped: 28212 kB\nSlab: 85324 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282236416}, {buffered_memory,252256256}, {free_memory,309755904}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{711919,1}}, {context_switches,{126153,0}}, {garbage_collection,{80280,491838291,0}}, {io,{{input,9538901},{output,6247139}}}, {reductions,{108026809,122578}}, {run_queue,0}, {runtime,{28650,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304247,729273}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35255096}, {processes,13839096}, {processes_used,13817776}, {system,21416000}, {atom,828329}, {atom_used,823498}, {binary,189416}, {code,8669006}, {ets,2277664}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,796}, {memory_data,{2058928128,1583640576,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 463404 kB\nBuffers: 352552 kB\nCached: 901464 kB\nSwapCached: 17552 kB\nActive: 765712 kB\nInactive: 631984 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 463404 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 384 kB\nWriteback: 0 kB\nAnonPages: 139424 kB\nMapped: 17648 kB\nSlab: 124596 kB\nPageTables: 6600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,923099136}, {buffered_memory,361013248}, {free_memory,474525696}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{791232,10}}, {context_switches,{200990,0}}, {garbage_collection,{57093,260187235,0}}, {io,{{input,19681117},{output,14870973}}}, {reductions,{106073010,1021678}}, {run_queue,0}, {runtime,{13190,140}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304247,894957}}, {active_buckets,[]}, {memory, [{total,29895480}, {processes,10025544}, {processes_used,10004072}, {system,19869936}, {atom,784585}, {atom_used,765874}, {binary,208344}, {code,7944128}, {ets,1613184}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,764}, {memory_data,{2058928128,1068896256,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 967200 kB\nBuffers: 252628 kB\nCached: 510776 kB\nSwapCached: 17896 kB\nActive: 678480 kB\nInactive: 229680 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 967200 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 208 kB\nWriteback: 0 kB\nAnonPages: 140192 kB\nMapped: 20308 kB\nSlab: 111504 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523034624}, {buffered_memory,258691072}, {free_memory,990412800}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{754806,2}}, {context_switches,{123511,0}}, {garbage_collection,{64155,421649743,0}}, {io,{{input,9853617},{output,7540820}}}, {reductions,{93459139,128770}}, {run_queue,0}, {runtime,{27370,30}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.955.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.959.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.959.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.961.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:10:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.961.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:10:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.964.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:10:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:10:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.964.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.966.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.966.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.969.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.969.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.971.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.971.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.974.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.974.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.976.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.976.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.979.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:11:31 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:11:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.979.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.981.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.981.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.984.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.984.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.986.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:11:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304307,652162}}, {active_buckets,[]}, {memory, [{total,15418776}, {processes,4855412}, {processes_used,4843532}, {system,10563364}, {atom,519117}, {atom_used,500490}, {binary,213400}, {code,4062708}, {ets,849588}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,777}, {memory_data,{2075373568,1765617664,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302868 kB\nBuffers: 246344 kB\nCached: 1251884 kB\nSwapCached: 0 kB\nActive: 737884 kB\nInactive: 887404 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 301232 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 156 kB\nWriteback: 0 kB\nAnonPages: 127092 kB\nMapped: 28212 kB\nSlab: 85328 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1281929216}, {buffered_memory,252256256}, {free_memory,310136832}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{771916,1}}, {context_switches,{131360,0}}, {garbage_collection,{81272,494494230,0}}, {io,{{input,10029896},{output,6528301}}}, {reductions,{108837909,147752}}, {run_queue,0}, {runtime,{28760,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304307,729210}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35349432}, {processes,13869768}, {processes_used,13848448}, {system,21479664}, {atom,828329}, {atom_used,823498}, {binary,163976}, {code,8669006}, {ets,2365056}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,856}, {memory_data,{2058928128,1585037312,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 462288 kB\nBuffers: 352552 kB\nCached: 902248 kB\nSwapCached: 17552 kB\nActive: 766324 kB\nInactive: 632536 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 462288 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 276 kB\nWriteback: 0 kB\nAnonPages: 139804 kB\nMapped: 17648 kB\nSlab: 124592 kB\nPageTables: 6600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 572096 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,923901952}, {buffered_memory,361013248}, {free_memory,473382912}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{851237,15}}, {context_switches,{211552,0}}, {garbage_collection,{60685,277060208,0}}, {io,{{input,19928135},{output,15713727}}}, {reductions,{112093375,1038359}}, {run_queue,0}, {runtime,{14060,140}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304307,895025}}, {active_buckets,[]}, {memory, [{total,29873576}, {processes,9974424}, {processes_used,9952952}, {system,19899152}, {atom,784585}, {atom_used,765874}, {binary,205208}, {code,7944128}, {ets,1644080}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,824}, {memory_data,{2058928128,1068896256,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966828 kB\nBuffers: 252628 kB\nCached: 510888 kB\nSwapCached: 17896 kB\nActive: 678540 kB\nInactive: 229672 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966828 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 196 kB\nWriteback: 0 kB\nAnonPages: 140136 kB\nMapped: 20308 kB\nSlab: 111500 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523149312}, {buffered_memory,258691072}, {free_memory,990031872}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{814806,2}}, {context_switches,{128351,0}}, {garbage_collection,{64802,424156433,0}}, {io,{{input,10094917},{output,7694106}}}, {reductions,{94220076,143894}}, {run_queue,0}, {runtime,{27520,30}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.986.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.990.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.990.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.992.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:11:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:11:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.992.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:11:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.995.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.439.0> 2011-04-08 16:11:58 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:11:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:11:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.995.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.997.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.997.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1000.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1000.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1002.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1002.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1005.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1005.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1007.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1007.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1010.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1010.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1012.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1012.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1015.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1015.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1017.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:12:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304367,652317}}, {active_buckets,[]}, {memory, [{total,15440176}, {processes,4856340}, {processes_used,4844460}, {system,10583836}, {atom,519117}, {atom_used,500490}, {binary,216600}, {code,4062708}, {ets,866028}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,837}, {memory_data,{2075373568,1765236736,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302744 kB\nBuffers: 246344 kB\nCached: 1252000 kB\nSwapCached: 0 kB\nActive: 737956 kB\nInactive: 887484 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 301108 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 80 kB\nWriteback: 0 kB\nAnonPages: 127112 kB\nMapped: 28212 kB\nSlab: 85324 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282048000}, {buffered_memory,252256256}, {free_memory,310009856}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{831918,1}}, {context_switches,{136200,0}}, {garbage_collection,{82108,496998802,0}}, {io,{{input,10286661},{output,6683473}}}, {reductions,{109575132,119093}}, {run_queue,0}, {runtime,{28850,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304367,729995}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35101872}, {processes,13490888}, {processes_used,13469568}, {system,21610984}, {atom,828329}, {atom_used,823498}, {binary,136488}, {code,8669006}, {ets,2520664}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,916}, {memory_data,{2058928128,1594585088,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 423016 kB\nBuffers: 352552 kB\nCached: 899560 kB\nSwapCached: 17552 kB\nActive: 765552 kB\nInactive: 630348 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 423016 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 244 kB\nWriteback: 0 kB\nAnonPages: 139540 kB\nMapped: 17648 kB\nSlab: 124720 kB\nPageTables: 6600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,921149440}, {buffered_memory,361013248}, {free_memory,433168384}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{911236,14}}, {context_switches,{222752,0}}, {garbage_collection,{64629,294090822,0}}, {io,{{input,22614429},{output,17852847}}}, {reductions,{118144189,993608}}, {run_queue,0}, {runtime,{15020,150}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304366,894398}}, {active_buckets,[]}, {memory, [{total,29886472}, {processes,10015776}, {processes_used,9994304}, {system,19870696}, {atom,784585}, {atom_used,765874}, {binary,208312}, {code,7944128}, {ets,1610872}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,883}, {memory_data,{2058928128,1068769280,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966952 kB\nBuffers: 252628 kB\nCached: 511008 kB\nSwapCached: 17896 kB\nActive: 678388 kB\nInactive: 229692 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966952 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 156 kB\nWriteback: 0 kB\nAnonPages: 139888 kB\nMapped: 20308 kB\nSlab: 111504 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523272192}, {buffered_memory,258691072}, {free_memory,990158848}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{874806,2}}, {context_switches,{133184,0}}, {garbage_collection,{65449,426561031,0}}, {io,{{input,10336296},{output,7854329}}}, {reductions,{94957751,121427}}, {run_queue,0}, {runtime,{27660,20}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1017.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1021.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:12:52 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:12:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1021.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1023.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:12:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1023.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:12:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1026.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:12:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:12:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1026.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1028.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1028.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1044.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1044.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1046.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.439.0> 2011-04-08 16:13:13 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:13:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1046.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1049.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1049.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1051.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1051.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1054.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1054.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1056.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1056.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1059.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1059.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1061.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:13:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304427,651419}}, {active_buckets,[]}, {memory, [{total,15417952}, {processes,4862732}, {processes_used,4850852}, {system,10555220}, {atom,519117}, {atom_used,500490}, {binary,209304}, {code,4062708}, {ets,843772}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,897}, {memory_data,{2075373568,1765363712,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302620 kB\nBuffers: 246344 kB\nCached: 1252108 kB\nSwapCached: 0 kB\nActive: 738008 kB\nInactive: 887556 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300984 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 188 kB\nWriteback: 0 kB\nAnonPages: 127112 kB\nMapped: 28212 kB\nSlab: 85320 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282158592}, {buffered_memory,252256256}, {free_memory,309882880}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{891915,1}}, {context_switches,{141103,0}}, {garbage_collection,{82985,499583395,0}}, {io,{{input,10544046},{output,6842055}}}, {reductions,{110344707,143679}}, {run_queue,0}, {runtime,{28960,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304427,729676}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,34585144}, {processes,12908376}, {processes_used,12887056}, {system,21676768}, {atom,828329}, {atom_used,823498}, {binary,182200}, {code,8669006}, {ets,2538976}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,976}, {memory_data,{2058928128,1583386624,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 464396 kB\nBuffers: 352552 kB\nCached: 900336 kB\nSwapCached: 17552 kB\nActive: 766020 kB\nInactive: 630896 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 464396 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 292 kB\nWriteback: 0 kB\nAnonPages: 139780 kB\nMapped: 17648 kB\nSlab: 124600 kB\nPageTables: 6600 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,921944064}, {buffered_memory,361013248}, {free_memory,475541504}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{971235,13}}, {context_switches,{233287,0}}, {garbage_collection,{68423,310241200,0}}, {io,{{input,22891694},{output,18688864}}}, {reductions,{124026181,1016518}}, {run_queue,0}, {runtime,{15880,140}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304427,894928}}, {active_buckets,[]}, {memory, [{total,29833128}, {processes,9927968}, {processes_used,9906496}, {system,19905160}, {atom,784585}, {atom_used,765874}, {binary,205896}, {code,7944128}, {ets,1645296}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,944}, {memory_data,{2058928128,1068896256,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966704 kB\nBuffers: 252628 kB\nCached: 511100 kB\nSwapCached: 17896 kB\nActive: 678404 kB\nInactive: 229860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966704 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 228 kB\nWriteback: 0 kB\nAnonPages: 139976 kB\nMapped: 20308 kB\nSlab: 111504 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523366400}, {buffered_memory,258691072}, {free_memory,989904896}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{934806,2}}, {context_switches,{138355,0}}, {garbage_collection,{66282,429059849,0}}, {io,{{input,10839253},{output,8233302}}}, {reductions,{95731989,120225}}, {run_queue,0}, {runtime,{27830,30}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1061.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1065.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1065.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1067.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:13:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1067.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:13:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1070.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:13:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:13:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:14:02 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:14:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1070.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1072.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1072.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1075.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1075.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1077.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1077.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1080.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1080.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1082.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1082.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1085.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1085.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1087.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1087.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1090.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:14:39 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:14:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1090.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1092.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:14:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304487,651499}}, {active_buckets,[]}, {memory, [{total,15384800}, {processes,4808596}, {processes_used,4796716}, {system,10576204}, {atom,519117}, {atom_used,500490}, {binary,211184}, {code,4062708}, {ets,861444}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,957}, {memory_data,{2075373568,1765490688,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302496 kB\nBuffers: 246352 kB\nCached: 1252212 kB\nSwapCached: 0 kB\nActive: 738024 kB\nInactive: 887660 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300860 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 184 kB\nWriteback: 0 kB\nAnonPages: 127116 kB\nMapped: 28212 kB\nSlab: 85324 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282265088}, {buffered_memory,252264448}, {free_memory,309755904}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{951917,1}}, {context_switches,{146298,0}}, {garbage_collection,{84085,502173891,0}}, {io,{{input,11032198},{output,7221087}}}, {reductions,{111118224,122129}}, {run_queue,0}, {runtime,{29080,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304487,728255}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35065144}, {processes,13206376}, {processes_used,13185056}, {system,21858768}, {atom,828329}, {atom_used,823498}, {binary,209192}, {code,8669006}, {ets,2692416}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1036}, {memory_data,{2058928128,1584517120,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 463340 kB\nBuffers: 352552 kB\nCached: 901108 kB\nSwapCached: 17552 kB\nActive: 766292 kB\nInactive: 631460 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 463340 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 432 kB\nWriteback: 0 kB\nAnonPages: 139836 kB\nMapped: 17648 kB\nSlab: 124664 kB\nPageTables: 6596 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 571508 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,922734592}, {buffered_memory,361013248}, {free_memory,474460160}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1031238,16}}, {context_switches,{243746,0}}, {garbage_collection,{72227,326242445,0}}, {io,{{input,23138685},{output,19525151}}}, {reductions,{129879067,992013}}, {run_queue,0}, {runtime,{16700,130}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304486,894419}}, {active_buckets,[]}, {memory, [{total,29750792}, {processes,9939104}, {processes_used,9917632}, {system,19811688}, {atom,784585}, {atom_used,765874}, {binary,207384}, {code,7944128}, {ets,1548680}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1003}, {memory_data,{2058928128,1069023232,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966400 kB\nBuffers: 252628 kB\nCached: 511216 kB\nSwapCached: 17896 kB\nActive: 678716 kB\nInactive: 229860 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966400 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 240 kB\nWriteback: 0 kB\nAnonPages: 140172 kB\nMapped: 20308 kB\nSlab: 111564 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 552200 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523485184}, {buffered_memory,258691072}, {free_memory,989593600}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{994806,2}}, {context_switches,{143221,0}}, {garbage_collection,{66953,431517942,0}}, {io,{{input,11080497},{output,8389620}}}, {reductions,{96491757,141841}}, {run_queue,0}, {runtime,{27980,30}}]}]}] INFO REPORT <0.818.0> 2011-04-08 16:14:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1092.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1096.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:14:49 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:14:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1096.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1098.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:14:54 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:14:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:14:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:14:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1098.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:14:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1101.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:14:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1101.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1103.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1103.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1106.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1106.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1108.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1108.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1111.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1111.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1113.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1113.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1116.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1116.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1118.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:15:34 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:15:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1118.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1121.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:15:40 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:15:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1121.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1123.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:15:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304547,650839}}, {active_buckets,[]}, {memory, [{total,15366456}, {processes,4839596}, {processes_used,4827716}, {system,10526860}, {atom,519117}, {atom_used,500490}, {binary,201968}, {code,4062708}, {ets,820380}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,1017}, {memory_data,{2075373568,1765617664,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302372 kB\nBuffers: 246352 kB\nCached: 1252328 kB\nSwapCached: 0 kB\nActive: 738092 kB\nInactive: 887700 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300736 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 48 kB\nWriteback: 0 kB\nAnonPages: 127120 kB\nMapped: 28212 kB\nSlab: 85328 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282383872}, {buffered_memory,252264448}, {free_memory,309628928}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{1011917,2}}, {context_switches,{151206,0}}, {garbage_collection,{84962,504666475,0}}, {io,{{input,11304250},{output,7378477}}}, {reductions,{111855031,120007}}, {run_queue,0}, {runtime,{29200,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304547,729967}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35287808}, {processes,13325512}, {processes_used,13304192}, {system,21962296}, {atom,828329}, {atom_used,823498}, {binary,153816}, {code,8669006}, {ets,2848304}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1096}, {memory_data,{2058928128,1587236864,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 453956 kB\nBuffers: 352560 kB\nCached: 908008 kB\nSwapCached: 17552 kB\nActive: 770628 kB\nInactive: 635356 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 453956 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 7176 kB\nWriteback: 0 kB\nAnonPages: 141168 kB\nMapped: 17652 kB\nSlab: 125340 kB\nPageTables: 6848 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 575952 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,929800192}, {buffered_memory,361021440}, {free_memory,464850944}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1091237,15}}, {context_switches,{255002,0}}, {garbage_collection,{76216,342733526,0}}, {io,{{input,25841471},{output,22607864}}}, {reductions,{135891679,1029271}}, {run_queue,0}, {runtime,{17630,140}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304546,894485}}, {active_buckets,[]}, {memory, [{total,29738992}, {processes,9895904}, {processes_used,9874432}, {system,19843088}, {atom,784585}, {atom_used,765874}, {binary,205992}, {code,7944128}, {ets,1580016}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1063}, {memory_data,{2058928128,1069150208,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966208 kB\nBuffers: 252632 kB\nCached: 511352 kB\nSwapCached: 17896 kB\nActive: 678840 kB\nInactive: 229864 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966208 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 1932 kB\nWriteback: 0 kB\nAnonPages: 140160 kB\nMapped: 20312 kB\nSlab: 111584 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 551836 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523624448}, {buffered_memory,258695168}, {free_memory,989396992}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1054806,2}}, {context_switches,{148084,0}}, {garbage_collection,{67629,434021641,0}}, {io,{{input,11367042},{output,8544783}}}, {reductions,{97238706,119099}}, {run_queue,0}, {runtime,{28110,20}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1123.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1127.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1127.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1129.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:15:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:15:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1129.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:15:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1132.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:15:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:15:58 =============================================================================== Pulling config from: 'ns_1@10.1.5.178' INFO REPORT <0.818.0> 2011-04-08 16:15:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1132.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1134.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1134.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1152.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1152.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1154.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1154.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1157.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1157.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1159.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:25 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:26 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:27 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:28 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1159.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:28 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1162.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:28 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:29 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:30 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:31 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:32 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:33 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1162.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:33 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1164.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:33 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:34 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:35 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:36 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:37 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:38 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1164.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:38 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1167.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:38 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:39 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:40 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:41 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:42 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:43 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1167.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:43 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1169.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:43 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:44 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:45 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:46 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:47 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.460.0> 2011-04-08 16:16:47 =============================================================================== ns_1@10.1.5.193:<0.460.0>:ns_doctor:86: Current node statuses: [{'ns_1@10.1.5.178', [{last_heard,{1302,304607,649135}}, {active_buckets,[]}, {memory, [{total,15379704}, {processes,4827780}, {processes_used,4815900}, {system,10551924}, {atom,519117}, {atom_used,500490}, {binary,209496}, {code,4062708}, {ets,837084}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"i686-pc-linux-gnu"}, {wall_clock,1077}, {memory_data,{2075373568,1765744640,{<10509.414.0>,786412}}}, {disk_data, [{"/",11173104,41},{"/boot",101086,13},{"/dev/shm",1013364,0}]}, {meminfo, <<"MemTotal: 2026732 kB\nMemFree: 302248 kB\nBuffers: 246352 kB\nCached: 1252440 kB\nSwapCached: 0 kB\nActive: 738156 kB\nInactive: 887760 kB\nHighTotal: 1130432 kB\nHighFree: 1636 kB\nLowTotal: 896300 kB\nLowFree: 300612 kB\nSwapTotal: 4063224 kB\nSwapFree: 4063224 kB\nDirty: 68 kB\nWriteback: 0 kB\nAnonPages: 127120 kB\nMapped: 28212 kB\nSlab: 85316 kB\nPageTables: 2256 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 5076588 kB\nCommitted_AS: 518936 kB\nVmallocTotal: 114680 kB\nVmallocUsed: 5620 kB\nVmallocChunk: 108816 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 4096 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2075373568}, {free_swap,4160741376}, {total_swap,4160741376}, {cached_memory,1282498560}, {buffered_memory,252264448}, {free_memory,309501952}, {total_memory,2075373568}]}, {statistics, [{wall_clock,{1071915,1}}, {context_switches,{156180,0}}, {garbage_collection,{85908,507362578,0}}, {io,{{input,11576707},{output,7534835}}}, {reductions,{112670218,144461}}, {run_queue,0}, {runtime,{29310,20}}]}]}, {'ns_1@10.1.5.192', [{last_heard,{1302,304607,730193}}, {active_buckets, ["FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561"]}, {memory, [{total,35385000}, {processes,13255600}, {processes_used,13234280}, {system,22129400}, {atom,828329}, {atom_used,823498}, {binary,166456}, {code,8669006}, {ets,3001152}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {inets,"5.2"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1156}, {memory_data,{2058928128,1594331136,{<10461.462.0>,2172304}}}, {disk_data, [{"/",13140720,42},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 453708 kB\nBuffers: 352560 kB\nCached: 908780 kB\nSwapCached: 17552 kB\nActive: 771108 kB\nInactive: 635888 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 453708 kB\nSwapTotal: 2031608 kB\nSwapFree: 1967216 kB\nDirty: 376 kB\nWriteback: 0 kB\nAnonPages: 141408 kB\nMapped: 17652 kB\nSlab: 124944 kB\nPageTables: 6852 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 575260 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2014429184}, {total_swap,2080366592}, {cached_memory,930590720}, {buffered_memory,361021440}, {free_memory,464596992}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1151236,14}}, {context_switches,{265521,0}}, {garbage_collection,{79985,358894690,0}}, {io,{{input,26118745},{output,23437046}}}, {reductions,{141766880,1014661}}, {run_queue,0}, {runtime,{18470,150}}]}]}, {'ns_1@10.1.5.193', [{last_heard,{1302,304606,894437}}, {active_buckets,[]}, {memory, [{total,29895600}, {processes,10004864}, {processes_used,9983392}, {system,19890736}, {atom,784585}, {atom_used,765874}, {binary,217576}, {code,7944128}, {ets,1613816}]}, {cluster_compatibility_version,1}, {version, [{kernel,"2.13.4"}, {mnesia,"4.4.12"}, {stdlib,"1.16.4"}, {os_mon,"2.2.4"}, {ns_server,"1.6.5.3_291_gba5fe27"}, {menelaus,"1.6.5.3_291_gba5fe27"}, {sasl,"2.1.8"}]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,1123}, {memory_data,{2058928128,1069404160,{<0.427.0>,1572784}}}, {disk_data, [{"/",13140720,39},{"/boot",101086,13},{"/dev/shm",1005336,0}]}, {meminfo, <<"MemTotal: 2010672 kB\nMemFree: 966456 kB\nBuffers: 252636 kB\nCached: 511044 kB\nSwapCached: 17896 kB\nActive: 678756 kB\nInactive: 229732 kB\nHighTotal: 0 kB\nHighFree: 0 kB\nLowTotal: 2010672 kB\nLowFree: 966456 kB\nSwapTotal: 2031608 kB\nSwapFree: 1984036 kB\nDirty: 144 kB\nWriteback: 0 kB\nAnonPages: 140248 kB\nMapped: 20312 kB\nSlab: 111532 kB\nPageTables: 5940 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nCommitLimit: 3036944 kB\nCommitted_AS: 551840 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 264508 kB\nVmallocChunk: 34359472391 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugepagesize: 2048 kB\n">>}, {replication, [{"FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561",1.0}]}, {system_memory_data, [{system_total_memory,2058928128}, {free_swap,2031652864}, {total_swap,2080366592}, {cached_memory,523309056}, {buffered_memory,258699264}, {free_memory,989650944}, {total_memory,2058928128}]}, {statistics, [{wall_clock,{1114806,2}}, {context_switches,{153324,0}}, {garbage_collection,{68425,436687704,0}}, {io,{{input,11872133},{output,8878638}}}, {reductions,{98095266,142139}}, {run_queue,0}, {runtime,{28300,20}}]}]}] SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:48 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1169.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:48 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1173.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:48 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:49 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:50 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:51 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.439.0> 2011-04-08 16:16:51 =============================================================================== Pulling config from: 'ns_1@10.1.5.192' INFO REPORT <0.818.0> 2011-04-08 16:16:52 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:53 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1173.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:53 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1175.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:53 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:54 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:55 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:56 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:57 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:16:58 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1175.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:16:58 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1178.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:16:58 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:16:59 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:00 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:01 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:02 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:17:03 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1178.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:17:03 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1180.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:17:03 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:04 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:05 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:06 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:07 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:17:08 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1180.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:17:08 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1183.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:17:08 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:09 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:10 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:11 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:12 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:17:13 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1183.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:17:13 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1185.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:17:13 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:14 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:15 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:16 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:17 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:17:18 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1185.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:17:18 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1188.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:17:18 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:19 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:20 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:21 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:22 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. SUPERVISOR REPORT <0.462.0> 2011-04-08 16:17:23 =============================================================================== Reporting supervisor {local,menelaus_sup} Child process errorContext child_terminated reason {noproc,{gen_server,call, ['ns_memcached-FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561', topkeys,30000]}} pid <0.1188.0> name hot_keys_keeper start_function {hot_keys_keeper,start_link,[]} restart_type permanent shutdown 5000 child_type worker PROGRESS REPORT <0.462.0> 2011-04-08 16:17:23 =============================================================================== supervisor {local,menelaus_sup} started [{pid,<0.1191.0>}, {name,hot_keys_keeper}, {mfa,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] INFO REPORT <0.818.0> 2011-04-08 16:17:23 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it. INFO REPORT <0.818.0> 2011-04-08 16:17:24 =============================================================================== ns_1@10.1.5.193:<0.818.0>:ns_memcached:443: Waiting for "FailoverNodesTests-4a448bb0-d65f-443f-9cd6-54b958201561" to finish shutting down before we start it.