[ns_doctor:debug,2014-03-06T15:45:03.110,ns_1@192.168.100.106:ns_doctor<0.1390.0>:ns_doctor:handle_info:166]Current node statuses: [{'ns_1@192.168.100.104', [{last_heard,{1394,91901,240045}}, {outgoing_replications_safeness_level, [{"usercenter",green},{"contacts",green},{"presence",green}]}, {incoming_replications_conf_hashes, [{"usercenter",[{'ns_1@192.168.100.107',102682548}]}, {"contacts",[{'ns_1@192.168.100.107',21715008}]}, {"presence", [{'ns_1@192.168.100.106',93102760}, {'ns_1@192.168.100.107',112729734}]}]}, {active_buckets,["usercenter","presence","contacts"]}, {ready_buckets,["usercenter","presence","contacts"]}, {local_tasks,[]}, {memory, [{total,861766704}, {processes,613276600}, {processes_used,607781864}, {system,248490104}, {atom,1302641}, {atom_used,1291173}, {binary,48843096}, {code,12748623}, {ets,172259384}]}, {system_memory_data, [{system_total_memory,16844836864}, {free_swap,8289509376}, {total_swap,8384409600}, {cached_memory,5239087104}, {buffered_memory,716546048}, {free_memory,1687302144}, {total_memory,16844836864}]}, {node_storage_conf, [{db_path,"/opt/couchbase/var/lib/couchbase/data"}, {index_path,"/opt/couchbase/var/lib/couchbase/data"}]}, {statistics, [{wall_clock,{14167204843,5003}}, {context_switches,{37673697811,0}}, {garbage_collection,{2258487958,35048941804783,0}}, {io,{{input,1378244614050},{output,19263149756470}}}, {reductions,{19942143388789,12733303}}, {run_queue,0}, {runtime,{2045339660,1130}}]}, {system_stats, [{cpu_utilization_rate,0.5835764902042517}, {swap_total,8384409600}, {swap_used,94900224}]}, {interesting_stats, [{couch_docs_actual_disk_size,135034046}, {couch_docs_data_size,134174904}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,3435}, {curr_items_tot,5566}, {mem_used,66149360}, {vb_replica_curr_items,2131}]}, {processes_stats, [{<<"proc/beam.smp/cpu_utilization">>,0}, {<<"proc/beam.smp/major_faults">>,0}, {<<"proc/beam.smp/major_faults_raw">>,0}, {<<"proc/beam.smp/mem_resident">>,1670844416}, {<<"proc/beam.smp/mem_share">>,42209280}, {<<"proc/beam.smp/mem_size">>,3875364864}, {<<"proc/beam.smp/minor_faults">>,64}, {<<"proc/beam.smp/minor_faults_raw">>,62117700390}, {<<"proc/beam.smp/page_faults">>,64}, {<<"proc/beam.smp/page_faults_raw">>,62117700390}, {<<"proc/inet_gethost/cpu_utilization">>,0}, {<<"proc/inet_gethost/major_faults">>,0}, {<<"proc/inet_gethost/major_faults_raw">>,0}, {<<"proc/inet_gethost/mem_resident">>,348160}, {<<"proc/inet_gethost/mem_share">>,258048}, {<<"proc/inet_gethost/mem_size">>,11096064}, {<<"proc/inet_gethost/minor_faults">>,0}, {<<"proc/inet_gethost/minor_faults_raw">>,600}, {<<"proc/inet_gethost/page_faults">>,0}, {<<"proc/inet_gethost/page_faults_raw">>,600}, {<<"proc/memcached/cpu_utilization">>,0}, {<<"proc/memcached/major_faults">>,0}, {<<"proc/memcached/major_faults_raw">>,3}, {<<"proc/memcached/mem_resident">>,178503680}, {<<"proc/memcached/mem_share">>,3203072}, {<<"proc/memcached/mem_size">>,505548800}, {<<"proc/memcached/minor_faults">>,0}, {<<"proc/memcached/minor_faults_raw">>,752683}, {<<"proc/memcached/page_faults">>,0}, {<<"proc/memcached/page_faults_raw">>,752686}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8ca6d2a"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-1969a70-git"}, {compiler,"4.7.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-1969a70-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-1969a70-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.1.0-718-rel-enterprise"}, {mochiweb,"2.4.2"}, {syntax_tools,"1.6.7.1"}, {xmerl,"1.2.10"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,14167204}, {memory_data,{16844836864,15148294144,{<15653.6.0>,102712680}}}, {disk_data, [{"/",100791728,12}, {"/dev/shm",8225016,0}, {"/boot",99150,55}, {"/data",726866688,1}, {"/home",604758736,1}]}, {meminfo, <<"MemTotal: 16450036 kB\nMemFree: 1647756 kB\nBuffers: 699752 kB\nCached: 5116296 kB\nSwapCached: 17092 kB\nActive: 8619304 kB\nInactive: 3877156 kB\nActive(anon): 5841404 kB\nInactive(anon): 880476 kB\nActive(file): 2777900 kB\nInactive(file): 2996680 kB\nUnevictable: 1628260 kB\nMlocked: 0 kB\nSwapTotal: 8187900 kB\nSwapFree: 8095224 kB\nDirty: 556 kB\nWriteback: 0 kB\nAnonPages: 8292208 kB\nMapped: 858336 kB\nShmem: 112 kB\nSlab: 427144 kB\nSReclaimable: 328028 kB\nSUnreclaim: 99116 kB\nKernelStack: 12424 kB\nPageTables: 39952 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 16412916 kB\nCommitted_AS: 10307288 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 307620 kB\nVmallocChunk: 34351021020 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 4304896 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 5604 kB\nDirectMap2M: 2078720 kB\nDirectMap1G: 14680064 kB\n">>}]}, {'ns_1@192.168.100.106', [{last_heard,{1394,91833,117386}}, {outgoing_replications_safeness_level, [{"usercenter",green},{"contacts",green},{"presence",green}]}, {incoming_replications_conf_hashes, [{"usercenter", [{'ns_1@192.168.100.104',8912284}, {'ns_1@192.168.100.107',56536436}]}, {"contacts", [{'ns_1@192.168.100.104',43325904}, {'ns_1@192.168.100.107',12754605}]}, {"presence", [{'ns_1@192.168.100.104',96925268}, {'ns_1@192.168.100.107',72478837}]}]}, {active_buckets,["presence","contacts","usercenter"]}, {ready_buckets,["presence","contacts","usercenter"]}, {local_tasks,[]}, {memory, [{total,1261600184}, {processes,1006202928}, {processes_used,1001977472}, {system,255397256}, {atom,1272633}, {atom_used,1250688}, {binary,47642984}, {code,12231608}, {ets,180433848}]}, {system_memory_data, [{system_total_memory,16844836864}, {free_swap,8268500992}, {total_swap,8384409600}, {cached_memory,6215774208}, {buffered_memory,418447360}, {free_memory,1081737216}, {total_memory,16844836864}]}, {node_storage_conf, [{db_path,"/opt/couchbase/var/lib/couchbase/data"}, {index_path,"/opt/couchbase/var/lib/couchbase/data"}]}, {statistics, [{wall_clock,{14165973795,5004}}, {context_switches,{35801984331,0}}, {garbage_collection,{2313130266,33807066153382,0}}, {io,{{input,1428563143250},{output,19282466607134}}}, {reductions,{16561732698507,5415529}}, {run_queue,0}, {runtime,{1344708880,370}}]}, {system_stats, [{cpu_utilization_rate,3.588748787584869}, {swap_total,8384409600}, {swap_used,115908608}]}, {interesting_stats, [{couch_docs_actual_disk_size,145705142}, {couch_docs_data_size,144848354}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2691}, {curr_items_tot,5515}, {mem_used,66291936}, {vb_replica_curr_items,2824}]}, {processes_stats, [{<<"proc/beam.smp/cpu_utilization">>,0}, {<<"proc/beam.smp/major_faults">>,0}, {<<"proc/beam.smp/major_faults_raw">>,0}, {<<"proc/beam.smp/mem_resident">>,1932161024}, {<<"proc/beam.smp/mem_share">>,42201088}, {<<"proc/beam.smp/mem_size">>,3736596480}, {<<"proc/beam.smp/minor_faults">>,3037}, {<<"proc/beam.smp/minor_faults_raw">>,63033936929}, {<<"proc/beam.smp/page_faults">>,3037}, {<<"proc/beam.smp/page_faults_raw">>,63033936929}, {<<"proc/inet_gethost/cpu_utilization">>,0}, {<<"proc/inet_gethost/major_faults">>,0}, {<<"proc/inet_gethost/major_faults_raw">>,8}, {<<"proc/inet_gethost/mem_resident">>,430080}, {<<"proc/inet_gethost/mem_share">>,413696}, {<<"proc/inet_gethost/mem_size">>,11096064}, {<<"proc/inet_gethost/minor_faults">>,0}, {<<"proc/inet_gethost/minor_faults_raw">>,625}, {<<"proc/inet_gethost/page_faults">>,0}, {<<"proc/inet_gethost/page_faults_raw">>,633}, {<<"proc/memcached/cpu_utilization">>,200}, {<<"proc/memcached/major_faults">>,0}, {<<"proc/memcached/major_faults_raw">>,34}, {<<"proc/memcached/mem_resident">>,164573184}, {<<"proc/memcached/mem_share">>,3002368}, {<<"proc/memcached/mem_size">>,499261440}, {<<"proc/memcached/minor_faults">>,0}, {<<"proc/memcached/minor_faults_raw">>,693965}, {<<"proc/memcached/page_faults">>,0}, {<<"proc/memcached/page_faults_raw">>,693999}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8ca6d2a"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-1969a70-git"}, {compiler,"4.7.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-1969a70-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-1969a70-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.1.0-718-rel-enterprise"}, {mochiweb,"2.4.2"}, {syntax_tools,"1.6.7.1"}, {xmerl,"1.2.10"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,14165973}, {memory_data,{16844836864,15756779520,{<0.16940.0>,19371648}}}, {disk_data, [{"/",100791728,38}, {"/dev/shm",8225016,0}, {"/boot",99150,55}, {"/home",371235760,1}]}, {meminfo, <<"MemTotal: 16450036 kB\nMemFree: 1056384 kB\nBuffers: 408640 kB\nCached: 6070092 kB\nSwapCached: 25344 kB\nActive: 8887832 kB\nInactive: 3890140 kB\nActive(anon): 5515932 kB\nInactive(anon): 824584 kB\nActive(file): 3371900 kB\nInactive(file): 3065556 kB\nUnevictable: 1884724 kB\nMlocked: 0 kB\nSwapTotal: 8187900 kB\nSwapFree: 8074708 kB\nDirty: 332 kB\nWriteback: 108 kB\nAnonPages: 8168956 kB\nMapped: 628592 kB\nShmem: 0 kB\nSlab: 473924 kB\nSReclaimable: 367428 kB\nSUnreclaim: 106496 kB\nKernelStack: 13040 kB\nPageTables: 42408 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 16412916 kB\nCommitted_AS: 11603964 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 307664 kB\nVmallocChunk: 34351018736 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 4067328 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 5604 kB\nDirectMap2M: 2078720 kB\nDirectMap1G: 14680064 kB\n">>}]}, {'ns_1@192.168.100.107', [{last_heard,{1394,91899,989160}}, {outgoing_replications_safeness_level, [{"usercenter",green},{"contacts",green},{"presence",green}]}, {incoming_replications_conf_hashes, [{"usercenter", [{'ns_1@192.168.100.104',20547089}, {'ns_1@192.168.100.106',112035787}]}, {"contacts", [{'ns_1@192.168.100.104',111274792}, {'ns_1@192.168.100.106',5172199}]}, {"presence", [{'ns_1@192.168.100.104',89202035}, {'ns_1@192.168.100.106',60975998}]}]}, {active_buckets,["usercenter","presence","contacts"]}, {ready_buckets,["usercenter","presence","contacts"]}, {local_tasks,[]}, {memory, [{total,704540480}, {processes,459251560}, {processes_used,457626168}, {system,245288920}, {atom,1252433}, {atom_used,1221274}, {binary,46073416}, {code,11716047}, {ets,172873440}]}, {system_memory_data, [{system_total_memory,16844836864}, {free_swap,8330952704}, {total_swap,8384409600}, {cached_memory,7849799680}, {buffered_memory,339574784}, {free_memory,391503872}, {total_memory,16844836864}]}, {node_storage_conf, [{db_path,"/opt/couchbase/var/lib/couchbase/data"}, {index_path,"/opt/couchbase/var/lib/couchbase/data"}]}, {statistics, [{wall_clock,{14013488172,5000}}, {context_switches,{33118186097,0}}, {garbage_collection,{2198662658,30800488409183,0}}, {io,{{input,1407652527111},{output,18616973760862}}}, {reductions,{12373079593291,5956562}}, {run_queue,0}, {runtime,{1793048590,450}}]}, {system_stats, [{cpu_utilization_rate,2.780117944397641}, {swap_total,8384409600}, {swap_used,53456896}]}, {interesting_stats, [{couch_docs_actual_disk_size,138765046}, {couch_docs_data_size,137922108}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,2702}, {curr_items_tot,5497}, {mem_used,65864656}, {vb_replica_curr_items,2795}]}, {processes_stats, [{<<"proc/beam.smp/cpu_utilization">>,0}, {<<"proc/beam.smp/major_faults">>,0}, {<<"proc/beam.smp/major_faults_raw">>,1}, {<<"proc/beam.smp/mem_resident">>,1254383616}, {<<"proc/beam.smp/mem_share">>,41406464}, {<<"proc/beam.smp/mem_size">>,2926313472}, {<<"proc/beam.smp/minor_faults">>,878}, {<<"proc/beam.smp/minor_faults_raw">>,47176494636}, {<<"proc/beam.smp/page_faults">>,878}, {<<"proc/beam.smp/page_faults_raw">>,47176494637}, {<<"proc/memcached/cpu_utilization">>,166}, {<<"proc/memcached/major_faults">>,0}, {<<"proc/memcached/major_faults_raw">>,71}, {<<"proc/memcached/mem_resident">>,149209088}, {<<"proc/memcached/mem_share">>,2998272}, {<<"proc/memcached/mem_size">>,461479936}, {<<"proc/memcached/minor_faults">>,0}, {<<"proc/memcached/minor_faults_raw">>,820047}, {<<"proc/memcached/page_faults">>,0}, {<<"proc/memcached/page_faults_raw">>,820118}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8ca6d2a"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-1969a70-git"}, {compiler,"4.7.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-1969a70-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-1969a70-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.1.0-718-rel-enterprise"}, {mochiweb,"2.4.2"}, {syntax_tools,"1.6.7.1"}, {xmerl,"1.2.10"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,14013488}, {memory_data,{16844836864,16449294336,{<15673.21623.19>,19371648}}}, {disk_data, [{"/",100791728,41}, {"/dev/shm",8225016,0}, {"/boot",99150,55}, {"/home",371235760,1}]}, {meminfo, <<"MemTotal: 16450036 kB\nMemFree: 382328 kB\nBuffers: 331616 kB\nCached: 7665820 kB\nSwapCached: 13760 kB\nActive: 10470948 kB\nInactive: 3689168 kB\nActive(anon): 5746960 kB\nInactive(anon): 456232 kB\nActive(file): 4723988 kB\nInactive(file): 3232936 kB\nUnevictable: 1218452 kB\nMlocked: 0 kB\nSwapTotal: 8187900 kB\nSwapFree: 8135696 kB\nDirty: 556 kB\nWriteback: 0 kB\nAnonPages: 7377420 kB\nMapped: 478404 kB\nShmem: 4 kB\nSlab: 450460 kB\nSReclaimable: 390128 kB\nSUnreclaim: 60332 kB\nKernelStack: 11368 kB\nPageTables: 36464 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 16412916 kB\nCommitted_AS: 10785452 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 307664 kB\nVmallocChunk: 34350992576 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 5718016 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 5604 kB\nDirectMap2M: 2078720 kB\nDirectMap1G: 14680064 kB\n">>}]}] [stats:warn,2014-03-06T15:45:04.945,ns_1@192.168.100.106:<0.16936.0>:stats_collector:latest_tick:240]Dropped 37 ticks [stats:warn,2014-03-06T15:45:04.952,ns_1@192.168.100.106:<0.26816.1>:stats_collector:latest_tick:240]Dropped 37 ticks [stats:warn,2014-03-06T15:45:04.953,ns_1@192.168.100.106:<0.1508.0>:stats_collector:latest_tick:240]Dropped 28 ticks [stats:error,2014-03-06T15:45:06.961,ns_1@192.168.100.106:<0.1508.0>:stats_collector:handle_info:106]Exception in stats collector: {exit, {{badmatch,{error,closed}}, {gen_server,call, ['ns_memcached-usercenter', {stats,<<>>}, 180000]}}, [{gen_server,call,3}, {ns_memcached,do_call,3}, {stats_collector,grab_all_stats,1}, {stats_collector,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [stats:error,2014-03-06T15:45:06.967,ns_1@192.168.100.106:<0.26816.1>:stats_collector:handle_info:106]Exception in stats collector: {exit, {{badmatch,{error,closed}}, {gen_server,call, ['ns_memcached-presence', {stats,<<>>}, 180000]}}, [{gen_server,call,3}, {ns_memcached,do_call,3}, {stats_collector,grab_all_stats,1}, {stats_collector,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [stats:error,2014-03-06T15:45:06.970,ns_1@192.168.100.106:<0.16936.0>:stats_collector:handle_info:106]Exception in stats collector: {exit, {{badmatch,{error,closed}}, {gen_server,call, ['ns_memcached-contacts', {stats,<<>>}, 180000]}}, [{gen_server,call,3}, {ns_memcached,do_call,3}, {stats_collector,grab_all_stats,1}, {stats_collector,handle_info,2}, {gen_server,handle_msg,5}, {proc_lib,init_p_do_apply,3}]} [stats:warn,2014-03-06T15:45:09.477,ns_1@192.168.100.106:<0.26816.1>:stats_collector:latest_tick:240]Dropped 2 ticks [ns_doctor:debug,2014-03-06T15:46:03.110,ns_1@192.168.100.106:ns_doctor<0.1390.0>:ns_doctor:handle_info:166]Current node statuses: [{'ns_1@192.168.100.104', [{last_heard,{1394,91961,239901}}, {outgoing_replications_safeness_level, [{"usercenter",green},{"contacts",green},{"presence",green}]}, {incoming_replications_conf_hashes, [{"usercenter",[{'ns_1@192.168.100.107',102682548}]}, {"contacts",[{'ns_1@192.168.100.107',21715008}]}, {"presence",[{'ns_1@192.168.100.107',112729734}]}]}, {active_buckets,["usercenter","presence","contacts"]}, {ready_buckets,["usercenter","presence","contacts"]}, {local_tasks,[]}, {memory, [{total,864357648}, {processes,614464768}, {processes_used,608945928}, {system,249892880}, {atom,1302641}, {atom_used,1291173}, {binary,49839400}, {code,12748623}, {ets,172672640}]}, {system_memory_data, [{system_total_memory,16844836864}, {free_swap,8289550336}, {total_swap,8384409600}, {cached_memory,5240193024}, {buffered_memory,716582912}, {free_memory,1688465408}, {total_memory,16844836864}]}, {node_storage_conf, [{db_path,"/opt/couchbase/var/lib/couchbase/data"}, {index_path,"/opt/couchbase/var/lib/couchbase/data"}]}, {statistics, [{wall_clock,{14167264842,5002}}, {context_switches,{37673883754,0}}, {garbage_collection,{2258500946,35049142964608,0}}, {io,{{input,1378254516823},{output,19263326484046}}}, {reductions,{19942249462072,10131299}}, {run_queue,0}, {runtime,{2045353190,580}}]}, {system_stats, [{cpu_utilization_rate,0.5008347245409015}, {swap_total,8384409600}, {swap_used,94859264}]}, {interesting_stats, [{couch_docs_actual_disk_size,136452755}, {couch_docs_data_size,135596216}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,4256}, {curr_items_tot,5566}, {mem_used,66148496}, {vb_replica_curr_items,1310}]}, {processes_stats, [{<<"proc/beam.smp/cpu_utilization">>,0}, {<<"proc/beam.smp/major_faults">>,0}, {<<"proc/beam.smp/major_faults_raw">>,0}, {<<"proc/beam.smp/mem_resident">>,1666510848}, {<<"proc/beam.smp/mem_share">>,42209280}, {<<"proc/beam.smp/mem_size">>,3870937088}, {<<"proc/beam.smp/minor_faults">>,238}, {<<"proc/beam.smp/minor_faults_raw">>,62118070513}, {<<"proc/beam.smp/page_faults">>,238}, {<<"proc/beam.smp/page_faults_raw">>,62118070513}, {<<"proc/inet_gethost/cpu_utilization">>,0}, {<<"proc/inet_gethost/major_faults">>,0}, {<<"proc/inet_gethost/major_faults_raw">>,0}, {<<"proc/inet_gethost/mem_resident">>,348160}, {<<"proc/inet_gethost/mem_share">>,258048}, {<<"proc/inet_gethost/mem_size">>,11096064}, {<<"proc/inet_gethost/minor_faults">>,0}, {<<"proc/inet_gethost/minor_faults_raw">>,600}, {<<"proc/inet_gethost/page_faults">>,0}, {<<"proc/inet_gethost/page_faults_raw">>,600}, {<<"proc/memcached/cpu_utilization">>,0}, {<<"proc/memcached/major_faults">>,0}, {<<"proc/memcached/major_faults_raw">>,3}, {<<"proc/memcached/mem_resident">>,178503680}, {<<"proc/memcached/mem_share">>,3203072}, {<<"proc/memcached/mem_size">>,505548800}, {<<"proc/memcached/minor_faults">>,0}, {<<"proc/memcached/minor_faults_raw">>,752684}, {<<"proc/memcached/page_faults">>,0}, {<<"proc/memcached/page_faults_raw">>,752687}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8ca6d2a"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-1969a70-git"}, {compiler,"4.7.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-1969a70-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-1969a70-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.1.0-718-rel-enterprise"}, {mochiweb,"2.4.2"}, {syntax_tools,"1.6.7.1"}, {xmerl,"1.2.10"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,14167264}, {memory_data,{16844836864,15153709056,{<15653.6.0>,102712680}}}, {disk_data, [{"/",100791728,12}, {"/dev/shm",8225016,0}, {"/boot",99150,55}, {"/data",726866688,1}, {"/home",604758736,1}]}, {meminfo, <<"MemTotal: 16450036 kB\nMemFree: 1648892 kB\nBuffers: 699788 kB\nCached: 5117376 kB\nSwapCached: 17072 kB\nActive: 8620036 kB\nInactive: 3877448 kB\nActive(anon): 5841240 kB\nInactive(anon): 880488 kB\nActive(file): 2778796 kB\nInactive(file): 2996960 kB\nUnevictable: 1626696 kB\nMlocked: 0 kB\nSwapTotal: 8187900 kB\nSwapFree: 8095264 kB\nDirty: 432 kB\nWriteback: 0 kB\nAnonPages: 8290332 kB\nMapped: 858336 kB\nShmem: 112 kB\nSlab: 426388 kB\nSReclaimable: 327992 kB\nSUnreclaim: 98396 kB\nKernelStack: 12400 kB\nPageTables: 39964 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 16412916 kB\nCommitted_AS: 10305268 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 307620 kB\nVmallocChunk: 34351021020 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 4304896 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 5604 kB\nDirectMap2M: 2078720 kB\nDirectMap1G: 14680064 kB\n">>}]}, {'ns_1@192.168.100.106', [{last_heard,{1394,91958,115508}}, {outgoing_replications_safeness_level,[]}, {incoming_replications_conf_hashes,[]}, {active_buckets,[]}, {ready_buckets,[]}, {local_tasks,[]}, {memory, [{total,496206392}, {processes,396311104}, {processes_used,391498936}, {system,99895288}, {atom,1272633}, {atom_used,1250740}, {binary,46604608}, {code,12231608}, {ets,26403008}]}, {system_memory_data, [{system_total_memory,16844836864}, {free_swap,8280870912}, {total_swap,8384409600}, {cached_memory,6223282176}, {buffered_memory,418742272}, {free_memory,1708761088}, {total_memory,16844836864}]}, {node_storage_conf, [{db_path,"/opt/couchbase/var/lib/couchbase/data"}, {index_path,"/opt/couchbase/var/lib/couchbase/data"}]}, {statistics, [{wall_clock,{14166098793,5000}}, {context_switches,{35802177728,0}}, {garbage_collection,{2313148684,33807244274821,0}}, {io,{{input,1428575216845},{output,19282630420337}}}, {reductions,{16561793972256,117228}}, {run_queue,0}, {runtime,{1344720950,30}}]}, {system_stats, [{cpu_utilization_rate,0.2085070892410342}, {swap_total,8384409600}, {swap_used,103538688}]}, {interesting_stats,[]}, {processes_stats, [{<<"proc/beam.smp/cpu_utilization">>,0}, {<<"proc/beam.smp/major_faults">>,0}, {<<"proc/beam.smp/major_faults_raw">>,0}, {<<"proc/beam.smp/mem_resident">>,1346605056}, {<<"proc/beam.smp/mem_share">>,42201088}, {<<"proc/beam.smp/mem_size">>,3150872576}, {<<"proc/beam.smp/minor_faults">>,0}, {<<"proc/beam.smp/minor_faults_raw">>,63034191150}, {<<"proc/beam.smp/page_faults">>,0}, {<<"proc/beam.smp/page_faults_raw">>,63034191150}, {<<"proc/inet_gethost/cpu_utilization">>,0}, {<<"proc/inet_gethost/major_faults">>,0}, {<<"proc/inet_gethost/major_faults_raw">>,8}, {<<"proc/inet_gethost/mem_resident">>,430080}, {<<"proc/inet_gethost/mem_share">>,413696}, {<<"proc/inet_gethost/mem_size">>,11096064}, {<<"proc/inet_gethost/minor_faults">>,0}, {<<"proc/inet_gethost/minor_faults_raw">>,625}, {<<"proc/inet_gethost/page_faults">>,0}, {<<"proc/inet_gethost/page_faults_raw">>,633}, {<<"proc/memcached/cpu_utilization">>,0}, {<<"proc/memcached/major_faults">>,0}, {<<"proc/memcached/major_faults_raw">>,15}, {<<"proc/memcached/mem_resident">>,86417408}, {<<"proc/memcached/mem_share">>,2977792}, {<<"proc/memcached/mem_size">>,266194944}, {<<"proc/memcached/minor_faults">>,0}, {<<"proc/memcached/minor_faults_raw">>,20868}, {<<"proc/memcached/page_faults">>,0}, {<<"proc/memcached/page_faults_raw">>,20883}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8ca6d2a"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-1969a70-git"}, {compiler,"4.7.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-1969a70-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-1969a70-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.1.0-718-rel-enterprise"}, {mochiweb,"2.4.2"}, {syntax_tools,"1.6.7.1"}, {xmerl,"1.2.10"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,14166098}, {memory_data,{16844836864,15135993856,{<0.1364.0>,13313560}}}, {disk_data, [{"/",100791728,38}, {"/dev/shm",8225016,0}, {"/boot",99150,55}, {"/home",371235760,1}]}, {meminfo, <<"MemTotal: 16450036 kB\nMemFree: 1668712 kB\nBuffers: 408928 kB\nCached: 6077424 kB\nSwapCached: 28096 kB\nActive: 8847892 kB\nInactive: 3891992 kB\nActive(anon): 5472772 kB\nInactive(anon): 822036 kB\nActive(file): 3375120 kB\nInactive(file): 3069956 kB\nUnevictable: 1310456 kB\nMlocked: 0 kB\nSwapTotal: 8187900 kB\nSwapFree: 8086788 kB\nDirty: 392 kB\nWriteback: 0 kB\nAnonPages: 7539260 kB\nMapped: 628904 kB\nShmem: 0 kB\nSlab: 470468 kB\nSReclaimable: 366396 kB\nSUnreclaim: 104072 kB\nKernelStack: 12760 kB\nPageTables: 41316 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 16412916 kB\nCommitted_AS: 10801612 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 307664 kB\nVmallocChunk: 34351018736 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 3700736 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 5604 kB\nDirectMap2M: 2078720 kB\nDirectMap1G: 14680064 kB\n">>}]}, {'ns_1@192.168.100.107', [{last_heard,{1394,91959,989235}}, {outgoing_replications_safeness_level, [{"usercenter",green},{"contacts",green},{"presence",green}]}, {incoming_replications_conf_hashes, [{"usercenter",[{'ns_1@192.168.100.104',20547089}]}, {"contacts",[{'ns_1@192.168.100.104',111274792}]}, {"presence",[{'ns_1@192.168.100.104',89202035}]}]}, {active_buckets,["usercenter","presence","contacts"]}, {ready_buckets,["usercenter","presence","contacts"]}, {local_tasks,[]}, {memory, [{total,707746776}, {processes,462245488}, {processes_used,460566408}, {system,245501288}, {atom,1252433}, {atom_used,1221274}, {binary,46730184}, {code,11716047}, {ets,172448880}]}, {system_memory_data, [{system_total_memory,16844836864}, {free_swap,8330993664}, {total_swap,8384409600}, {cached_memory,7852802048}, {buffered_memory,339681280}, {free_memory,380043264}, {total_memory,16844836864}]}, {node_storage_conf, [{db_path,"/opt/couchbase/var/lib/couchbase/data"}, {index_path,"/opt/couchbase/var/lib/couchbase/data"}]}, {statistics, [{wall_clock,{14013548172,5001}}, {context_switches,{33118372984,0}}, {garbage_collection,{2198683128,30800692859505,0}}, {io,{{input,1407660772033},{output,18617100711497}}}, {reductions,{12373159504939,5789756}}, {run_queue,0}, {runtime,{1793061210,410}}]}, {system_stats, [{cpu_utilization_rate,0.70862859524802}, {swap_total,8384409600}, {swap_used,53415936}]}, {interesting_stats, [{couch_docs_actual_disk_size,140867876}, {couch_docs_data_size,140027452}, {couch_views_actual_disk_size,0}, {couch_views_data_size,0}, {curr_items,4033}, {curr_items_tot,5497}, {mem_used,65862992}, {vb_replica_curr_items,1464}]}, {processes_stats, [{<<"proc/beam.smp/cpu_utilization">>,0}, {<<"proc/beam.smp/major_faults">>,0}, {<<"proc/beam.smp/major_faults_raw">>,1}, {<<"proc/beam.smp/mem_resident">>,1257648128}, {<<"proc/beam.smp/mem_share">>,41406464}, {<<"proc/beam.smp/mem_size">>,2929647616}, {<<"proc/beam.smp/minor_faults">>,1738}, {<<"proc/beam.smp/minor_faults_raw">>,47176823774}, {<<"proc/beam.smp/page_faults">>,1738}, {<<"proc/beam.smp/page_faults_raw">>,47176823775}, {<<"proc/memcached/cpu_utilization">>,0}, {<<"proc/memcached/major_faults">>,0}, {<<"proc/memcached/major_faults_raw">>,71}, {<<"proc/memcached/mem_resident">>,149209088}, {<<"proc/memcached/mem_share">>,2998272}, {<<"proc/memcached/mem_size">>,461479936}, {<<"proc/memcached/minor_faults">>,0}, {<<"proc/memcached/minor_faults_raw">>,820047}, {<<"proc/memcached/page_faults">>,0}, {<<"proc/memcached/page_faults_raw">>,820118}]}, {cluster_compatibility_version,131072}, {version, [{public_key,"0.13"}, {lhttpc,"1.3.0"}, {ale,"8ca6d2a"}, {os_mon,"2.2.7"}, {couch_set_view,"1.2.0a-1969a70-git"}, {compiler,"4.7.5"}, {inets,"5.7.1"}, {couch,"1.2.0a-1969a70-git"}, {mapreduce,"1.0.0"}, {couch_index_merger,"1.2.0a-1969a70-git"}, {kernel,"2.14.5"}, {crypto,"2.0.4"}, {ssl,"4.1.6"}, {sasl,"2.1.10"}, {couch_view_parser,"1.0.0"}, {ns_server,"2.1.0-718-rel-enterprise"}, {mochiweb,"2.4.2"}, {syntax_tools,"1.6.7.1"}, {xmerl,"1.2.10"}, {oauth,"7d85d3ef"}, {stdlib,"1.17.5"}]}, {supported_compat_version,[2,0]}, {system_arch,"x86_64-unknown-linux-gnu"}, {wall_clock,14013548}, {memory_data,{16844836864,16461221888,{<15673.21623.19>,19371648}}}, {disk_data, [{"/",100791728,41}, {"/dev/shm",8225016,0}, {"/boot",99150,55}, {"/home",371235760,1}]}, {meminfo, <<"MemTotal: 16450036 kB\nMemFree: 371136 kB\nBuffers: 331720 kB\nCached: 7668752 kB\nSwapCached: 14184 kB\nActive: 10473992 kB\nInactive: 3689556 kB\nActive(anon): 5746952 kB\nInactive(anon): 456696 kB\nActive(file): 4727040 kB\nInactive(file): 3232860 kB\nUnevictable: 1225116 kB\nMlocked: 0 kB\nSwapTotal: 8187900 kB\nSwapFree: 8135736 kB\nDirty: 596 kB\nWriteback: 0 kB\nAnonPages: 7384128 kB\nMapped: 478404 kB\nShmem: 4 kB\nSlab: 450664 kB\nSReclaimable: 390184 kB\nSUnreclaim: 60480 kB\nKernelStack: 11360 kB\nPageTables: 36464 kB\nNFS_Unstable: 0 kB\nBounce: 0 kB\nWritebackTmp: 0 kB\nCommitLimit: 16412916 kB\nCommitted_AS: 10792468 kB\nVmallocTotal: 34359738367 kB\nVmallocUsed: 307664 kB\nVmallocChunk: 34350992576 kB\nHardwareCorrupted: 0 kB\nAnonHugePages: 5718016 kB\nHugePages_Total: 0\nHugePages_Free: 0\nHugePages_Rsvd: 0\nHugePages_Surp: 0\nHugepagesize: 2048 kB\nDirectMap4k: 5604 kB\nDirectMap2M: 2078720 kB\nDirectMap1G: 14680064 kB\n">>}]}] [ns_doctor:debug,2014-03-06T15:47:03.110,ns_1@192.168.100.106:ns_doctor<0.1390.0>:ns_doctor:handle_info:166]Current node statuses: