=PROGRESS REPORT==== 14-May-2012::13:56:18 === supervisor: {local,sasl_safe_sup} started: [{pid,<0.31.0>}, {name,alarm_handler}, {mfargs,{alarm_handler,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] =PROGRESS REPORT==== 14-May-2012::13:56:18 === supervisor: {local,sasl_safe_sup} started: [{pid,<0.32.0>}, {name,overload}, {mfargs,{overload,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] =PROGRESS REPORT==== 14-May-2012::13:56:18 === supervisor: {local,sasl_sup} started: [{pid,<0.30.0>}, {name,sasl_safe_sup}, {mfargs, {supervisor,start_link, [{local,sasl_safe_sup},sasl,safe]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] =PROGRESS REPORT==== 14-May-2012::13:56:18 === supervisor: {local,sasl_sup} started: [{pid,<0.33.0>}, {name,release_handler}, {mfargs,{release_handler,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] =PROGRESS REPORT==== 14-May-2012::13:56:18 === application: sasl started_at: nonode@nohost [ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:<0.67.0>:ns_server:init_logging:138] Started & configured logging [ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:<0.67.0>:ns_server:log_pending:30] Static config terms: [{error_logger_mf_dir,"C:\\Program Files\\Couchbase\\Server\\var\\lib\\couchbase\\logs"}, {error_logger_mf_maxbytes,10485760}, {error_logger_mf_maxfiles,20}, {path_config_bindir,"C:\\Program Files\\Couchbase\\Server\\bin"}, {path_config_etcdir,"C:\\Program Files\\Couchbase\\Server\\etc\\couchbase"}, {path_config_libdir,"C:\\Program Files\\Couchbase\\Server\\lib"}, {path_config_datadir,"C:\\Program Files\\Couchbase\\Server\\var\\lib\\couchbase"}, {path_config_tmpdir,"C:\\Program Files\\Couchbase\\Server\\var\\lib\\couchbase\\tmp"}, {loglevel_default,info}, {loglevel_ns_server,info}, {loglevel_error_logger,info}, {loglevel_user,info}, {loglevel_menelaus,info}, {loglevel_ns_doctor,info}, {loglevel_stats,info}, {loglevel_rebalance,info}, {loglevel_cluster,info}] [ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:ns_server_cluster_sup:log_os_info:start_link:25] OS type: {win32,nt} Version: {6,1,7600} Runtime info: [{otp_release,"R14B02"}, {erl_version,"5.8.3"}, {erl_version_long, "Erlang R14B02 (erts-5.8.3) [smp:4:4] [rq:4] [async-threads:16]\n"}, {system_arch_raw,"win32"}, {system_arch,"windows"}, {localtime,{{2012,5,14},{13,37,16}}}, {memory, [{total,6256136}, {processes,898408}, {processes_used,890112}, {system,5357728}, {atom,446233}, {atom_used,438204}, {binary,15896}, {code,3414830}, {ets,193220}]}, {loaded, [ns_info,log_os_info,cb_init_loggers,ns_server_cluster_sup, ale_default_formatter,otp_internal,misc, 'ale_logger-cluster','ale_logger-rebalance', 'ale_logger-stats','ale_logger-ns_doctor', 'ale_logger-menelaus','ale_logger-user', 'ale_logger-ns_server',ns_log_sink,disk_log_sup, disk_log_server,disk_log_1,disk_log,ale_disk_sink, ns_server,timer,io_lib_fread,memsup,disksup, os_mon_sysinfo,os_mon,sasl_report,release_handler, calendar,overload,alarm_handler,sasl_report_tty_h,sasl, ale_error_logger_handler,'ale_logger-ale_logger', 'ale_logger-error_logger',beam_opcodes,beam_dict,beam_asm, beam_validator,beam_flatten,beam_trim,beam_receive, beam_bsm,beam_peep,beam_dead,beam_type,beam_bool, beam_clean,beam_utils,beam_jump,beam_block,v3_codegen, v3_life,v3_kernel,sys_core_dsetel,erl_bifs,sys_core_fold, cerl_trees,sys_core_inline,core_lib,cerl,v3_core,erl_bits, erl_expand_records,sys_pre_expand,sofs,erl_internal, compile,dynamic_compile,ale_utils,io_lib_pretty, ale_codegen,ale,ale_dynamic_sup,ale_sup,ale_app, io_lib_format,io_lib,io,sets,dict,ordsets,erl_lint, ram_file,beam_lib,ns_bootstrap,file_io_server,orddict, erl_eval,file,c,error_logger_tty_h,kernel_config,queue, shell,user,user_drv,user_sup,supervisor_bridge, standard_error,unicode,binary,ets,gb_sets, hipe_unified_loader,packages,code_server,code,file_server, global_group,gen_tcp,inet_tcp,auth,erl_epmd,inet_tcp_dist, net_kernel,erl_distribution,filename,win32reg,inet_parse, inet,inet_udp,os,inet_config,inet_db,global,gb_trees,rpc, supervisor,kernel,application_master,sys,application, gen_server,erl_parse,proplists,erl_scan,lists, application_controller,proc_lib,gen,gen_event, error_logger,heart,error_handler,erlang,erl_prim_loader, prim_zip,zlib,prim_file,prim_inet,init,otp_ring0]}, {applications, [{ale,"Another Logger for Erlang","8cffe61"}, {os_mon,"CPO CXC 138 46","2.2.5"}, {kernel,"ERTS CXC 138 10","2.14.3"}, {sasl,"SASL CXC 138 11","2.1.9.3"}, {ns_server,"Couchbase server","1.8.1-815-rel-enterprise"}, {stdlib,"ERTS CXC 138 10","1.17.3"}]}, {pre_loaded, [erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet, init,otp_ring0]}, {process_count,63}, {node,'ns_1@10.1.3.83'}, {nodes,[]}, {registered, [init,'sink-disk_debug','sink-disk_error',inet_db,rex, net_sup,kernel_sup,global_name_server,ale_sup, kernel_safe_sup,release_handler,file_server_2, ale_dynamic_sup,overload,global_group,alarm_handler, 'sink-disk_default',erl_prim_loader,erl_epmd, sasl_safe_sup,auth,error_logger,net_kernel,disk_log_sup, disk_log_server,os_mon_sup,ns_server_cluster_sup, code_server,os_mon_sysinfo,application_controller,memsup, disksup,standard_error,ale,timer_server, standard_error_sup,sasl_sup,user,'sink-ns_log']}, {cookie,nocookie}, {wordsize,4}, {wall_clock,1}] [ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:ns_server_cluster_sup:log_os_info:start_link:27] Manifest: ["","", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " ", " "," ", " ", " ", " ",""] [error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.128.0>}, {name,timeout_diag_logger}, {mfargs,{timeout_diag_logger,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:read_address_config:52] reading ip config from "c:/Program Files/Couchbase/Server/var/lib/couchbase/ip" [ns_server:error] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:read_address_config:58] Got error:einval. Ignoring bad address:[] [ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:bringup:145] Attempting to bring up net_kernel with name 'ns_1@127.0.0.1' [ns_server:debug] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:bringup:153] Attempted to save node name to disk: undefined [error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.129.0>}, {name,dist_manager}, {mfargs,{dist_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.130.0>}, {name,ns_cookie_manager}, {mfargs,{ns_cookie_manager,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.131.0>}, {name,ns_cluster}, {mfargs,{ns_cluster,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_mnesia_sup} started: [{pid,<0.133.0>}, {name,mb_mnesia_events}, {mfargs, {gen_event,start_link,[{local,mb_mnesia_events}]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.140.0>}, {name,mnesia_event}, {mfargs,{mnesia_sup,start_event,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.142.0>}, {name,mnesia_monitor}, {mfargs,{mnesia_monitor,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.143.0>}, {name,mnesia_subscr}, {mfargs,{mnesia_subscr,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.144.0>}, {name,mnesia_locker}, {mfargs,{mnesia_locker,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.145.0>}, {name,mnesia_recover}, {mfargs,{mnesia_recover,start,[]}}, {restart_type,permanent}, {shutdown,180000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.146.0>}, {name,mnesia_tm}, {mfargs,{mnesia_tm,start,[]}}, {restart_type,permanent}, {shutdown,30000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.147.0>}, {name,mnesia_checkpoint_sup}, {mfargs,{mnesia_checkpoint_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.148.0>}, {name,mnesia_snmp_sup}, {mfargs,{mnesia_snmp_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.149.0>}, {name,mnesia_controller}, {mfargs,{mnesia_controller,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_kernel_sup} started: [{pid,<0.150.0>}, {name,mnesia_late_loader}, {mfargs,{mnesia_late_loader,start,[]}}, {restart_type,permanent}, {shutdown,3000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mnesia_sup} started: [{pid,<0.141.0>}, {name,mnesia_kernel_sup}, {mfargs,{mnesia_kernel_sup,start,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: mnesia started_at: 'ns_1@10.1.3.83' [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_safe_sup} started: [{pid,<0.156.0>}, {name,dets_sup}, {mfargs,{dets_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_safe_sup} started: [{pid,<0.157.0>}, {name,dets}, {mfargs,{dets_server,start_link,[]}}, {restart_type,permanent}, {shutdown,2000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:ensure_schema:410] Committed schema to disk. [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:init:246] Current config: [{access_module,mnesia}, {auto_repair,true}, {backup_module,mnesia_backup}, {checkpoints,[]}, {db_nodes,['ns_1@10.1.3.83']}, {debug,verbose}, {directory,"c:/Program Files/Couchbase/Server/var/lib/couchbase/mnesia"}, {dump_log_load_regulation,false}, {dump_log_time_threshold,180000}, {dump_log_update_in_place,true}, {dump_log_write_threshold,1000}, {embedded_mnemosyne,false}, {event_module,mnesia_event}, {extra_db_nodes,[]}, {fallback_activated,false}, {held_locks,[]}, {ignore_fallback_at_startup,false}, {fallback_error_function,{mnesia,lkill}}, {is_running,yes}, {local_tables,[local_config,cluster,schema]}, {lock_queue,[]}, {log_version,"4.3"}, {master_node_tables,[]}, {max_wait_for_decision,10000}, {protocol_version,{7,6}}, {running_db_nodes,['ns_1@10.1.3.83']}, {schema_location,opt_disc}, {schema_version,{3,0}}, {subscribers,[<0.140.0>,<0.134.0>]}, {tables,[local_config,cluster,schema]}, {transaction_commits,5}, {transaction_failures,0}, {transaction_log_writes,3}, {transaction_restarts,0}, {transactions,[]}, {use_dir,true}, {core_dir,false}, {no_table_loaders,2}, {dc_dump_limit,4}, {send_compressed,0}, {version,"4.4.17"}] Peers: ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:167] Info from Mnesia: Create Directory "c:/Program Files/Couchbase/Server/var/lib/couchbase/mnesia" [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_mnesia_sup} started: [{pid,<0.134.0>}, {name,mb_mnesia}, {mfargs,{mb_mnesia,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,schema, [{name,schema}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,false}, {record_name,schema}, {attributes,[table,cstruct]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,114000},'ns_1@10.1.3.83'}}, {version,{{3,0},{'ns_1@10.1.3.83',{1337,27837,177002}}}}]}, {tid,3,<0.153.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:195] Peers: ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:195] Peers: ['ns_1@10.1.3.83'] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.132.0>}, {name,mb_mnesia_sup}, {mfargs,{mb_mnesia_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,333001},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,5,<0.176.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,local_config, [{name,local_config}, {type,set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,local_config}, {attributes,[key,val]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,333001},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,5,<0.176.0>}} [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_sup:ns_config_sup:init:32] loading static ns_config from "c:/Program Files/Couchbase/Server/etc/couchbase/config" [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.183.0>}, {name,ns_config_events}, {mfargs, {gen_event,start_link,[{local,ns_config_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:load_config:614] Loading static config from "c:/Program Files/Couchbase/Server/etc/couchbase/config" [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:load_config:628] Loading dynamic config from "c:/Program Files/Couchbase/Server/var/lib/couchbase/config/config.dat" [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:load_config:632] No dynamic config file found. Assuming we're brand new node [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_6_to_1_7:278] Upgrading config from 1.6 to 1.7 [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes: [{set,{node,'ns_1@10.1.3.83',config_version},{1,7}}, {set, {node,'ns_1@10.1.3.83',memcached}, [{port,11210}, {dedicated_port,11209}, {dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine, "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine, "c:/Program Files/Couchbase/Server/lib/memcached/ep.so"}, {initfile, "c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine, "c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {set, {node,'ns_1@10.1.3.83',ns_log}, [{filename, "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}]}, {set, {node,'ns_1@10.1.3.83',isasl}, [{path, "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}]}, {set,directory,"c:/Program Files/Couchbase/Server/var/lib/couchbase/config"}] [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_7_to_1_7_1:301] Upgrading config from 1.7 to 1.7.1 [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes: [{set,{node,'ns_1@10.1.3.83',config_version},{1,7,1}}, {set,email_alerts, [{recipients,["root@localhost"]}, {sender,"couchbase@localhost"}, {enabled,false}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {set,auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}] [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_7_1_to_1_7_2:312] Upgrading config from 1.7.1 to 1.7.2 [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes: [{set,{node,'ns_1@10.1.3.83',config_version},{1,7,2}}] [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_7_2_to_1_8_0:366] Upgrading config from 1.7.2 to 1.8.0 [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes: [{set,{node,'ns_1@10.1.3.83',config_version},{1,8,0}}, {set,{node,'ns_1@10.1.3.83',port_servers}, [{moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{misc,this_node_rest_port,[]}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR", {"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"c:/Program Files/Couchbase/Server/bin/memcached", ["-X", "c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so", "-l", {"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]}, "-E", "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}]}] [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_8_0_to_1_8_1:403] Upgrading config from 1.8.0 to 1.8.1 [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes: [{set,{node,'ns_1@10.1.3.83',config_version},{1,8,1}}, {set,vbucket_map_history,[]}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_init:465] Upgraded initial config: {config, {full,"c:/Program Files/Couchbase/Server/etc/couchbase/config",undefined, ns_config_default}, [[], [{directory,"c:/Program Files/Couchbase/Server/var/lib/couchbase/config"}, {nodes_wanted,['ns_1@10.1.3.83']}, {{node,'ns_1@10.1.3.83',membership},active}, {rest,[{port,8091}]}, {{node,'ns_1@10.1.3.83',rest},[{port,8091},{port_meta,global}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@10.1.3.83',isasl}, [{path, "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}]}, {{node,'ns_1@10.1.3.83',memcached}, [{port,11210}, {dedicated_port,11209}, {dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine, "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine, "c:/Program Files/Couchbase/Server/lib/memcached/ep.so"}, {initfile, "c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine, "c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {memory_quota,3276}, {buckets,[{configs,[]}]}, {{node,'ns_1@10.1.3.83',moxi},[{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.3.83',port_servers}, [{moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{misc,this_node_rest_port,[]}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"c:/Program Files/Couchbase/Server/bin/memcached", ["-X", "c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so", "-l", {"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]}, "-E", "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {{node,'ns_1@10.1.3.83',ns_log}, [{filename, "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}]}, {email_alerts, [{recipients,["root@localhost"]}, {sender,"couchbase@localhost"}, {enabled,false}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {replication,[{enabled,true}]}, {auto_failover_cfg, [{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}]], [[{vbucket_map_history,[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}]}, {directory, [{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}, 99,58,47,80,114,111,103,114,97,109,32,70,105,108,101,115,47,67,111, 117,99,104,98,97,115,101,47,83,101,114,118,101,114,47,118,97,114, 47,108,105,98,47,99,111,117,99,104,98,97,115,101,47,99,111,110,102, 105,103]}, {{node,'ns_1@10.1.3.83',config_version}, [{'_vclock',[{'ns_1@10.1.3.83',{5,63504247037}}]}|{1,8,1}]}, {auto_failover_cfg, [{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}, {enabled,false}, {timeout,30}, {max_nodes,1}, {count,0}]}, {buckets,[{configs,[]}]}, {email_alerts, [{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}, {recipients,["root@localhost"]}, {sender,"couchbase@localhost"}, {enabled,false}, {email_server, [{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts, [auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down, auto_failover_cluster_too_small]}]}, {memory_quota,3276}, {nodes_wanted,['ns_1@10.1.3.83']}, {replication,[{enabled,true}]}, {rest,[{port,8091}]}, {rest_creds,[{creds,[]}]}, {{node,'ns_1@10.1.3.83',isasl}, [{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}, {path, "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}]}, {{node,'ns_1@10.1.3.83',membership},active}, {{node,'ns_1@10.1.3.83',memcached}, [{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}, {port,11210}, {dedicated_port,11209}, {dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine, "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine, "c:/Program Files/Couchbase/Server/lib/memcached/ep.so"}, {initfile, "c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine, "c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}]}, {{node,'ns_1@10.1.3.83',moxi},[{port,11211},{verbosity,[]}]}, {{node,'ns_1@10.1.3.83',ns_log}, [{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}, {filename, "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}]}, {{node,'ns_1@10.1.3.83',port_servers}, [{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}, {moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{misc,this_node_rest_port,[]}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD", {"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout, stream]}, {memcached,"c:/Program Files/Couchbase/Server/bin/memcached", ["-X", "c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so", "-l", {"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]}, "-E", "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}]}, {{node,'ns_1@10.1.3.83',rest},[{port,8091},{port_meta,global}]}]], ns_config_default, {ns_config,save_config_sync,[]}, undefined,false} [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.184.0>}, {name,ns_config}, {mfargs, {ns_config,start_link, ["c:/Program Files/Couchbase/Server/etc/couchbase/config", ns_config_default]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.187.0>}, {name,ns_config_remote}, {mfargs, {ns_config_replica,start_link, [{local,ns_config_remote}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:init:49] isasl_sync init: ["c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw", "_admin","_admin"] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:init:57] isasl_sync init buckets: [] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw" [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.188.0>}, {name,ns_config_isasl_sync}, {mfargs,{ns_config_isasl_sync,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_config_sup} started: [{pid,<0.191.0>}, {name,ns_config_log}, {mfargs,{ns_config_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.182.0>}, {name,ns_config_sup}, {mfargs,{ns_config_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.193.0>}, {name,vbucket_filter_changes_registry}, {mfargs, {ns_process_registry,start_link, [vbucket_filter_changes_registry]}}, {restart_type,permanent}, {shutdown,100}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_log:ns_log:init:64] Couldn't load logs from "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log" (perhaps it's first startup): {error, enoent} [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.195.0>}, {name,ns_log}, {mfargs,{ns_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.196.0>}, {name,ns_log_events}, {mfargs,{gen_event,start_link,[{local,ns_log_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.198.0>}, {name,ns_node_disco_events}, {mfargs, {gen_event,start_link, [{local,ns_node_disco_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_node_disco:ns_node_disco:init:98] Initting ns_node_disco with [] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_sync:115] ns_cookie_manager do_cookie_sync [user:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_init:91] Initial otp cookie generated: caonwcesmjjqmzcs [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: otp -> [{cookie,caonwcesmjjqmzcs}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.200.0>:ns_node_disco:do_nodes_wanted_updated_fun:182] ns_node_disco: nodes_wanted updated: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.200.0>:ns_node_disco:do_nodes_wanted_updated_fun:188] ns_node_disco: nodes_wanted pong: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.199.0>}, {name,ns_node_disco}, {mfargs,{ns_node_disco,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.203.0>}, {name,ns_node_disco_log}, {mfargs,{ns_node_disco_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.204.0>}, {name,ns_node_disco_conf_events}, {mfargs,{ns_node_disco_conf_events,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:init:58] init pulling [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:init:60] init pushing [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:init:64] init reannouncing [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:50] ns_node_disco_conf_events config on otp [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: otp -> [{cookie,caonwcesmjjqmzcs}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_sync:115] ns_cookie_manager do_cookie_sync [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:44] ns_node_disco_conf_events config on nodes_wanted [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: vbucket_map_history -> [] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: directory -> "c:/Program Files/Couchbase/Server/var/lib/couchbase/config" [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.207.0>:ns_node_disco:do_nodes_wanted_updated_fun:182] ns_node_disco: nodes_wanted updated: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_sync:115] ns_cookie_manager do_cookie_sync [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.207.0>:ns_node_disco:do_nodes_wanted_updated_fun:188] ns_node_disco: nodes_wanted pong: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',config_version} -> {1,8,1} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: auto_failover_cfg -> [{enabled,false},{timeout,30},{max_nodes,1},{count,0}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.208.0>:ns_node_disco:do_nodes_wanted_updated_fun:182] ns_node_disco: nodes_wanted updated: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[]}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.208.0>:ns_node_disco:do_nodes_wanted_updated_fun:188] ns_node_disco: nodes_wanted pong: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: email_alerts -> [{recipients,["root@localhost"]}, {sender,"couchbase@localhost"}, {enabled,false}, {email_server,[{user,[]}, {pass,[]}, {host,"localhost"}, {port,25}, {encrypt,false}]}, {alerts,[auto_failover_node,auto_failover_maximum_reached, auto_failover_other_nodes_down,auto_failover_cluster_too_small]}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: memory_quota -> 3276 [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: nodes_wanted -> ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: replication -> [{enabled,true}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: rest -> [{port,8091}] [ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:handle_info:57] config change: rest_creds -> ******** [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',isasl} -> [{path,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',membership} -> active [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',memcached} -> [{port,11210}, {dedicated_port,11209}, {dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"}, {admin_user,"_admin"}, {admin_pass,"_admin"}, {bucket_engine, "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"}, {engines, [{membase, [{engine,"c:/Program Files/Couchbase/Server/lib/memcached/ep.so"}, {initfile, "c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"}, {static_config_string, "vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]}, {memcached, [{engine, "c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"}, {static_config_string,"vb0=true"}]}]}, {verbosity,[]}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',moxi} -> [{port,11211},{verbosity,[]}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',ns_log} -> [{filename,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',port_servers} -> [{moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", {"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", [port]}, "-z", {"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming", [{misc,this_node_rest_port,[]}]}, "-p","0","-Y","y","-O","stderr", {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}}, {"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]}, use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]}, {memcached,"c:/Program Files/Couchbase/Server/bin/memcached", ["-X", "c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so", "-l", {"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]}, "-E", "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", {"admin=~s;default_bucket_name=default;auto_create=false", [admin_user]}, {"~s",[verbosity]}], [{env,[{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE",{"~s",[{isasl,path}]}}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status,port_server_send_eol, stream]}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: {node,'ns_1@10.1.3.83',rest} -> [{port,8091},{port_meta,global}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_node_disco_sup} started: [{pid,<0.205.0>}, {name,ns_config_rep}, {mfargs,{ns_config_rep,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.197.0>}, {name,ns_node_disco_sup}, {mfargs,{ns_node_disco_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.210.0>}, {name,ns_tick_event}, {mfargs,{gen_event,start_link,[{local,ns_tick_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.211.0>}, {name,mb_master_events}, {mfargs, {gen_event,start_link,[{local,mb_master_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_server_sup:mb_master:check_master_takeover_needed:150] Was unable to discover master, not going to force mastership takeover [user:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master:mb_master:init:89] I'm the only node, so I'm the master. [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master_sup:misc:start_singleton:824] start_singleton(gen_fsm, ns_orchestrator, [], []): started as <0.218.0> on 'ns_1@10.1.3.83' [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.218.0>}, {name,ns_orchestrator}, {mfargs,{ns_orchestrator,start_link,[]}}, {restart_type,permanent}, {shutdown,20}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master_sup:misc:start_singleton:824] start_singleton(gen_server, ns_tick, [], []): started as <0.219.0> on 'ns_1@10.1.3.83' [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.219.0>}, {name,ns_tick}, {mfargs,{ns_tick,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.220.0>:auto_failover:init:120] init auto_failover. [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master_sup:misc:start_singleton:824] start_singleton(gen_server, auto_failover, [], []): started as <0.220.0> on 'ns_1@10.1.3.83' [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,mb_master_sup} started: [{pid,<0.220.0>}, {name,auto_failover}, {mfargs,{auto_failover,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.215.0>}, {name,mb_master}, {mfargs,{mb_master,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.221.0>}, {name,master_activity_events}, {mfargs, {gen_event,start_link, [{local,master_activity_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.222.0>}, {name,master_activity_events_ingress}, {mfargs, {gen_event,start_link, [{local,master_activity_events_ingress}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.223.0>}, {name,master_activity_events_timestamper}, {mfargs, {master_activity_events,start_link_timestamper,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.224.0>}, {name,master_activity_events_keeper}, {mfargs,{master_activity_events_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.226.0>}, {name,master_activity_events_srv}, {mfargs, {master_activity_events,start_link_work_server,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.227.0>}, {name,buckets_events}, {mfargs, {gen_event,start_link,[{local,buckets_events}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_mail_sup} started: [{pid,<0.229.0>}, {name,ns_mail}, {mfargs,{ns_mail,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_log_events:ns_mail_log:init:45] ns_mail_log started up [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_mail_sup} started: [{pid,<0.230.0>}, {name,ns_mail_log}, {mfargs,{ns_mail_log,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.228.0>}, {name,ns_mail_sup}, {mfargs,{ns_mail_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.231.0>}, {name,ns_heart}, {mfargs,{ns_heart,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.235.0>}, {name,ns_doctor}, {mfargs,{ns_doctor,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:144] Failed to grab system stats: {'EXIT',{noproc,{gen_server,call, [{'stats_reader-@system','ns_1@10.1.3.83'}, {latest,"minute"}]}}} [ns_doctor:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:89] Got initial status [{'ns_1@10.1.3.83', [{last_heard,{1337,27837,708000}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,9768936}, {processes,2306208}, {processes_used,2300160}, {system,7462728}, {atom,641577}, {atom_used,636189}, {binary,78184}, {code,5148278}, {ets,297676}]}, {system_stats, [{cpu_utilization_rate,0}, {swap_total,0}, {swap_used,0}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,2}, {memory_data,{4294500352,697974784,{<0.7.0>,159012}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3631923200}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{2656,812}}, {context_switches,{20357,0}}, {garbage_collection,{5231,18586032,0}}, {io,{{input,5464203},{output,772365}}}, {reductions,{10646300,10646300}}, {run_queue,0}, {runtime,{2593,2593}}]}]}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.238.0>}, {name,menelaus_web}, {mfargs,{menelaus_web,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.240.0>}, {name,menelaus_event}, {mfargs,{menelaus_event,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.241.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [user:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_server_sup:menelaus_sup:start_link:44] Couchbase Server has started on web port 8091 on node 'ns_1@10.1.3.83'. [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.242.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.237.0>}, {name,menelaus}, {mfargs,{menelaus_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.244.0>}, {name,ns_port_init}, {mfargs,{ns_port_init,start_link,[]}}, {restart_type,permanent}, {shutdown,10}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.245.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000 [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.247.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000 [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.245.0>}, {name, {moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.247.0>}, {name, {memcached, "c:/Program Files/Couchbase/Server/bin/memcached", ["-X", "c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so", "-l","0.0.0.0:11210,0.0.0.0:11209:1000","-E", "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]}}, {mfargs, {supervisor_cushion,start_link, [memcached,5000,ns_port_server,start_link, [memcached, "c:/Program Files/Couchbase/Server/bin/memcached", ["-X", "c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so", "-l","0.0.0.0:11210,0.0.0.0:11209:1000", "-E", "c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so", "-B","binary","-r","-c","10000","-e", "admin=_admin;default_bucket_name=default;auto_create=false", []], [{env, [{"EVENT_NOSELECT","1"}, {"MEMCACHED_TOP_KEYS","100"}, {"ISASL_PWFILE", "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}, {"ISASL_DB_CHECK_TIME","1"}]}, use_stdio,stderr_to_stdout,exit_status, port_server_send_eol,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.243.0>}, {name,ns_port_sup}, {mfargs,{ns_port_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.249.0>}, {name,ns_stats_event}, {mfargs, {gen_event,start_link,[{local,ns_stats_event}]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.250.0>}, {name,ns_bucket_worker}, {mfargs,{work_queue,start_link,[ns_bucket_worker]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.252.0>}, {name,buckets_observing_subscription}, {mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.251.0>}, {name,ns_bucket_sup}, {mfargs,{ns_bucket_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.253.0>}, {name,system_stats_collector}, {mfargs,{system_stats_collector,start_link,[]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.255.0>}, {name,{stats_archiver,"@system"}}, {mfargs,{stats_archiver,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-minute' [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.257.0>}, {name,{stats_reader,"@system"}}, {mfargs,{stats_reader,start_link,["@system"]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.258.0>}, {name,ns_moxi_sup_work_queue}, {mfargs, {work_queue,start_link,[ns_moxi_sup_work_queue]}}, {restart_type,permanent}, {shutdown,1000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_sup} started: [{pid,<0.259.0>}, {name,ns_moxi_sup}, {mfargs,{ns_moxi_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_server_cluster_sup} started: [{pid,<0.194.0>}, {name,ns_server_sup}, {mfargs,{ns_server_sup,start_link,[]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= application: ns_server started_at: 'ns_1@10.1.3.83' [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,848027},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,6,<0.261.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-minute', [{name,'stats_archiver-@system-minute'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,848027},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,6,<0.261.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-hour' [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,848040},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,7,<0.269.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-hour', [{name,'stats_archiver-@system-hour'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,848040},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,7,<0.269.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-day' [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,864005},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,8,<0.276.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-day', [{name,'stats_archiver-@system-day'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,864005},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,8,<0.276.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-week' [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,864014},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,9,<0.283.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-week', [{name,'stats_archiver-@system-week'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,864014},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,9,<0.283.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-month' [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,880007},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,10,<0.290.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-month', [{name,'stats_archiver-@system-month'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,880007},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,10,<0.290.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-year' [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,895005},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,11,<0.297.0>}} [ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event: {write,{schema,'stats_archiver-@system-year', [{name,'stats_archiver-@system-year'}, {type,ordered_set}, {ram_copies,[]}, {disc_copies,['ns_1@10.1.3.83']}, {disc_only_copies,[]}, {load_order,0}, {access_mode,read_write}, {index,[]}, {snmp,[]}, {local_content,true}, {record_name,stat_entry}, {attributes,[timestamp,values]}, {user_properties,[]}, {frag_properties,[]}, {cookie,{{1337,27837,895005},'ns_1@10.1.3.83'}}, {version,{{2,0},[]}}]}, {tid,11,<0.297.0>}} [ns_server:debug] [2012-05-14 13:37:18] [ns_1@10.1.3.83:<0.239.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61452 path /pools/default/saslBucketsStreaming [error_logger:info] [2012-05-14 13:37:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,inet_gethost_native_sup} started: [{pid,<0.308.0>},{mfa,{inet_gethost_native,init,[[]]}}] [error_logger:info] [2012-05-14 13:37:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,kernel_safe_sup} started: [{pid,<0.307.0>}, {name,inet_gethost_native_sup}, {mfargs,{inet_gethost_native,start_link,[]}}, {restart_type,temporary}, {shutdown,1000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_port_sup:terminate_port:87] unsupervising port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR",[]}, {"MOXI_SASL_PLAIN_PWD",[]}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} [ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:<0.246.0>:ns_port_server:log:161] moxi<0.246.0>: EOL on stdin. Exiting [menelaus:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:<0.239.0>:menelaus_web:handle_streaming:707] menelaus_web streaming socket closed by client [ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_port_sup:launch_port:54] supervising port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:<0.393.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000 [error_logger:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.393.0>}, {name, {moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: rest -> [{port,8091}] [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_log:ns_config_log:handle_info:57] config change: rest_creds -> ******** [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: memory_quota -> 3276 [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:debug] [2012-05-14 13:37:53] [ns_1@10.1.3.83:<0.397.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61516 path /pools/default/saslBucketsStreaming [ns_server:info] [2012-05-14 13:37:53] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: 2012-05-14 13:37:53: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.394.0>: 2012-05-14 13:37:53: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) [ns_doctor:debug] [2012-05-14 13:38:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,27892,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,11723872}, {processes,3704776}, {processes_used,3691432}, {system,8019096}, {atom,696501}, {atom_used,669919}, {binary,154664}, {code,5484890}, {ets,381220}]}, {system_stats, [{cpu_utilization_rate,25.0}, {swap_total,9850605568}, {swap_used,748527616}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,57}, {memory_data,{4294500352,697974784,{<0.7.0>,159012}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3642757120}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{47656,0}}, {context_switches,{28995,0}}, {garbage_collection,{6928,22353488,0}}, {io,{{input,7300017},{output,2437947}}}, {reductions,{11996122,220631}}, {run_queue,0}, {runtime,{3843,281}}]}]}] [ns_doctor:debug] [2012-05-14 13:39:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,27952,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,11642400}, {processes,3610708}, {processes_used,3597364}, {system,8031692}, {atom,696501}, {atom_used,669919}, {binary,149800}, {code,5484890}, {ets,398700}]}, {system_stats, [{cpu_utilization_rate,25.03173394262503}, {swap_total,9850605568}, {swap_used,745025536}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,117}, {memory_data,{4294500352,650600448,{<0.231.0>,300620}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3648258048}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{107656,0}}, {context_switches,{33168,0}}, {garbage_collection,{7540,23817698,0}}, {io,{{input,7309230},{output,2518291}}}, {reductions,{12481563,129508}}, {run_queue,0}, {runtime,{4890,250}}]}]}] [ns_doctor:debug] [2012-05-14 13:40:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28012,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,11910000}, {processes,3851912}, {processes_used,3838568}, {system,8058088}, {atom,696501}, {atom_used,669919}, {binary,160312}, {code,5484890}, {ets,414524}]}, {system_stats, [{cpu_utilization_rate,24.993649987299975}, {swap_total,9850605568}, {swap_used,745504768}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,177}, {memory_data,{4294500352,645046272,{<0.231.0>,300620}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3650203648}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{167656,0}}, {context_switches,{37410,0}}, {garbage_collection,{8036,25360744,0}}, {io,{{input,7318452},{output,2602984}}}, {reductions,{12982172,129654}}, {run_queue,0}, {runtime,{6031,328}}]}]}] [ns_doctor:debug] [2012-05-14 13:41:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28072,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,11986384}, {processes,3918620}, {processes_used,3904796}, {system,8067764}, {atom,696501}, {atom_used,669919}, {binary,153184}, {code,5484890}, {ets,431460}]}, {system_stats, [{cpu_utilization_rate,31.378522467631377}, {swap_total,9850605568}, {swap_used,754688000}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,237}, {memory_data,{4294500352,642867200,{<0.397.0>,300760}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3655122944}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{227656,0}}, {context_switches,{41980,0}}, {garbage_collection,{8641,26977742,0}}, {io,{{input,7571009},{output,2902350}}}, {reductions,{13522125,128746}}, {run_queue,0}, {runtime,{7343,312}}]}]}] [ns_doctor:debug] [2012-05-14 13:42:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28132,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,12834560}, {processes,4494112}, {processes_used,4483536}, {system,8340448}, {atom,700405}, {atom_used,677771}, {binary,330912}, {code,5554827}, {ets,448932}]}, {system_stats, [{cpu_utilization_rate,25.374650749301498}, {swap_total,9850605568}, {swap_used,771563520}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,297}, {memory_data,{4294500352,644669440,{<0.231.0>,300620}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3637673984}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{287656,0}}, {context_switches,{48329,0}}, {garbage_collection,{9609,30315266,0}}, {io,{{input,8937383},{output,4372830}}}, {reductions,{14585061,691292}}, {run_queue,0}, {runtime,{8609,469}}]}]}] [ns_doctor:debug] [2012-05-14 13:43:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28192,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,12342456}, {processes,4022036}, {processes_used,4009476}, {system,8320420}, {atom,700405}, {atom_used,677771}, {binary,297656}, {code,5554827}, {ets,464220}]}, {system_stats, [{cpu_utilization_rate,25.38109756097561}, {swap_total,9850605568}, {swap_used,766648320}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,357}, {memory_data,{4294500352,656515072,{<0.397.0>,300760}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3640819712}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{347656,0}}, {context_switches,{53069,0}}, {garbage_collection,{10197,33026302,0}}, {io,{{input,8946864},{output,4463522}}}, {reductions,{15167674,130303}}, {run_queue,0}, {runtime,{10000,407}}]}]}] [menelaus:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.1274.0>:menelaus_web_buckets:do_bucket_create:262] Created bucket "default" of type: membase [{num_replicas,1},{ram_quota,1048576000},{auth_type,sasl}] [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[[{map,[]}, {fastForwardMap,[]}, {sasl_password,[]}, {num_replicas,1}, {ram_quota,1048576000}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,[]}]]}] [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw" [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:84] Starting new child: {{per_bucket_sup,"default"}, {single_bucket_sup,start_link,["default"]}, permanent,infinity,supervisor, [single_bucket_sup]} [ns_server:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[{"default", [{map,[]}, {fastForwardMap,[]}, {sasl_password,[]}, {num_replicas,1}, {ram_quota,1048576000}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.3.83']}]}]}] [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [error_logger:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1315.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: 2012-05-14 13:44:07: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Empty serverList ({ moxi<0.394.0>: "name": "default", moxi<0.394.0>: "nodeLocator": "vbucket", moxi<0.394.0>: "saslPassword": "", moxi<0.394.0>: "nodes": [], moxi<0.394.0>: "vBucketServerMap": { moxi<0.394.0>: "hashAlgorithm": "CRC", moxi<0.394.0>: "numReplicas": 1, moxi<0.394.0>: "serverList": [], moxi<0.394.0>: "vBucketMap": [] moxi<0.394.0>: } moxi<0.394.0>: }) moxi<0.394.0>: 2012-05-14 13:44:07: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.394.0>: "name": "default", moxi<0.394.0>: "nodeLocator": "vbucket", moxi<0.394.0>: "saslPassword": "", moxi<0.394.0>: "nodes": [{ moxi<0.394.0>: "replication": "undefined", moxi<0.394.0>: "clusterMembership": "active", moxi<0.394.0>: "status": "warmup", moxi<0.394.0>: "hostname": "10.1.3.83:8091", moxi<0.394.0>: "clusterCompatibility": 1, moxi<0.394.0>: "version": "1.8.1-815-rel-enterprise", moxi<0.394.0>: "os": "windows", moxi<0.394.0>: "ports": { moxi<0.394.0>: "proxy": 11211, moxi<0.394.0>: "direct": 11210 moxi<0.394.0>: } moxi<0.394.0>: }], moxi<0.394.0>: "vBucketServerMap": { moxi<0.394.0>: "hashAlgorithm": "CRC", moxi<0.394.0>: "numReplicas": 1, moxi<0.394.0>: "serverList": ["10.1.3.83:11210"], moxi<0.394.0>: "vBucketMap": [] moxi<0.394.0>: } moxi<0.394.0>: }) [stats:error] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [menelaus:warn] [2012-05-14 13:44:05] [ns_1@10.1.3.83:<0.972.0>:menelaus_web:loop:273] Client-side error-report for user "Administrator" on node 'ns_1@10.1.3.83': User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 Got unhandled error: Uncaught Error: BUG At: http://10.1.3.83:8091/js/cells.js:64 Backtrace: Function: collectBacktraceViaCaller Args: --------- Function: appOnError Args: "Uncaught Error: BUG" "http://10.1.3.83:8091/js/cells.js" 64 --------- [ns_server:warn] [2012-05-14 13:44:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.242.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1329.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:44:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1307.0>} [ns_server:warn] [2012-05-14 13:44:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:44:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:44:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1329.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1334.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:09] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:44:10] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1334.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1346.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:44:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:info] [2012-05-14 13:44:12] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: 2012-05-14 13:44:15: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.394.0>: "name": "default", moxi<0.394.0>: "nodeLocator": "vbucket", moxi<0.394.0>: "saslPassword": "", moxi<0.394.0>: "nodes": [{ moxi<0.394.0>: "replication": 1, moxi<0.394.0>: "clusterMembership": "active", moxi<0.394.0>: "status": "warmup", moxi<0.394.0>: "hostname": "10.1.3.83:8091", moxi<0.394.0>: "clusterCompatibility": 1, moxi<0.394.0>: "version": "1.8.1-815-rel-enterprise", moxi<0.394.0>: "os": "windows", moxi<0.394.0>: "ports": { moxi<0.394.0>: "proxy": 11211, moxi<0.394.0>: "direct": 11210 moxi<0.394.0>: } moxi<0.394.0>: }], moxi<0.394.0>: "vBucketServerMap": { moxi<0.394.0>: "hashAlgorithm": "CRC", moxi<0.394.0>: "numReplicas": 1, moxi<0.394.0>: "serverList": ["10.1.3.83:11210"], moxi<0.394.0>: "vBucketMap": [] moxi<0.394.0>: } moxi<0.394.0>: }) [stats:error] [2012-05-14 13:44:12] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1317.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1315.0>] messages: [check_started,check_started, {'$gen_call',{<0.1325.0>, {#Ref<0.0.0.13178>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.13278>},connected}, {'$gen_call',{<0.1332.0>,#Ref<0.0.0.13280>},topkeys}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.1344.0>, {#Ref<0.0.0.13480>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.13554>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.1316.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7355 neighbours: [error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1315.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1315.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1355.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.241.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1356.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1346.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1362.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:15] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:44:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1307.0>} [ns_server:error] [2012-05-14 13:44:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_doctor:debug] [2012-05-14 13:44:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28254,723008}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13236424}, {processes,4925688}, {processes_used,4914864}, {system,8310736}, {atom,701869}, {atom_used,679949}, {binary,238368}, {code,5581197}, {ets,485348}]}, {system_stats, [{cpu_utilization_rate,26.618345413646587}, {swap_total,9850605568}, {swap_used,765308928}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,419}, {memory_data,{4294500352,653561856,{<0.231.0>,300620}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3641167872}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{407656,0}}, {context_switches,{58372,0}}, {garbage_collection,{10903,36345610,0}}, {io,{{input,9202417},{output,4639563}}}, {reductions,{15864473,207793}}, {run_queue,0}, {runtime,{11421,328}}]}]}] [error_logger:error] [2012-05-14 13:44:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1362.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1375.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:18] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:44:20] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:44:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1375.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1384.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:21] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:44:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:44:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1384.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1399.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:<0.1274.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1360.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1355.0>] messages: [check_started,check_started, {'$gen_call',{<0.1365.0>, {#Ref<0.0.0.13807>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.13955>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.1382.0>,#Ref<0.0.0.14103>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.1387.0>, {#Ref<0.0.0.14160>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.14290>},connected}, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.1357.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1355.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1403.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1356.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1405.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1355.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [stats:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1399.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1416.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:44:27] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1307.0>} [ns_server:error] [2012-05-14 13:44:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:44:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:28] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:44:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1416.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1426.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:31] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:44:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:44:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1426.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1437.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:34] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:35] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1406.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1403.0>] messages: [{'$gen_call',{<0.1307.0>,#Ref<0.0.0.14410>}, list_vbuckets_prevstate}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.14625>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.1427.0>,#Ref<0.0.0.14771>},topkeys}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.14927>},connected}, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.1404.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7350 neighbours: [error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.1307.0> registered_name: [] exception exit: {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4298 neighbours: [error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1403.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1403.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1447.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1405.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1450.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1437.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1455.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:37] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:44:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:44:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1455.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1466.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:40] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:44:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1466.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1477.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:44:42] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:44:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:44:43] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:44] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1477.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:44] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1488.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:44:45] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1449.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1447.0>] messages: [check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.1464.0>, {#Ref<0.0.0.15285>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.15309>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.1469.0>,#Ref<0.0.0.15372>},topkeys}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.15557>},connected}, check_started,check_started, {'$gen_call',{<0.1486.0>, {#Ref<0.0.0.15658>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started] links: [<0.64.0>,<0.1448.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7349 neighbours: [error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1447.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1447.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1489.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1450.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1492.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:46] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:44:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1462.0>} [ns_server:error] [2012-05-14 13:44:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:44:47] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1488.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:47] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1505.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:49] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1505.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1514.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:44:51] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:44:52] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:44:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:44:53] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1514.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:53] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1529.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:44:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:44:55] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1491.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1489.0>] messages: [check_started,check_started, {'$gen_call',{<0.1497.0>, {#Ref<0.0.0.15771>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.15910>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.1512.0>,#Ref<0.0.0.16052>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.1517.0>, {#Ref<0.0.0.16109>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.16235>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.1490.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1492.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1489.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1535.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1534.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1489.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [ns_server:warn] [2012-05-14 13:44:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:56] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1529.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:56] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1544.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:44:57] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1462.0>} [ns_server:error] [2012-05-14 13:44:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:44:58] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:44:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:44:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1544.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:44:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1554.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:45:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:45:01] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:45:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:45:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:45:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1554.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:45:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1565.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.1280.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:45:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:45:05] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1537.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1535.0>] messages: [check_started,check_started, {'$gen_call',{<0.1542.0>, {#Ref<0.0.0.16457>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.16524>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.1555.0>,#Ref<0.0.0.16670>},topkeys}, check_started,check_started, {'$gen_call',{<0.1462.0>,#Ref<0.0.0.16691>}, list_vbuckets_prevstate}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.16827>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.1536.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7354 neighbours: [error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.1462.0> registered_name: [] exception exit: {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4289 neighbours: [error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1535.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1534.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1535.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1585.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1584.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1565.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1598.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:45:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:45:07] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:45:07] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:45:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:45:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:45:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1598.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:45:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1611.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:45:09] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason shutdown [ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:91] Stopping child for dead bucket: {{per_bucket_sup,"default"}, <0.1585.0>,supervisor, [single_bucket_sup]} [ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[]}] [ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw" [ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:warn] [2012-05-14 13:45:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:45:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:45:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1585.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: shutdown_error Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1585.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1587.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1585.0>] messages: [check_started,check_started,check_started,check_started, {'$gen_call',{<0.1609.0>, {#Ref<0.0.0.17358>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.17380>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.17697>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.1586.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7344 neighbours: [ns_doctor:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28315,645003}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,13309552}, {processes,4937016}, {processes_used,4925752}, {system,8372536}, {atom,702357}, {atom_used,681053}, {binary,267416}, {code,5596508}, {ets,501260}]}, {system_stats, [{cpu_utilization_rate,27.368157960509873}, {swap_total,9850605568}, {swap_used,764198912}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,480}, {memory_data,{4294500352,653295616,{<0.972.0>,300780}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3643469824}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{470484,0}}, {context_switches,{68919,0}}, {garbage_collection,{12196,46153944,0}}, {io,{{input,9261751},{output,5005366}}}, {reductions,{17345273,293759}}, {run_queue,0}, {runtime,{14453,594}}]}]}] [ns_server:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:idle:373] Restarting moxi on nodes ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1618.0>:ns_pubsub:do_subscribe_link:132] Parent process exited with reason normal [ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1618.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.17517>} event handler: ok [ns_server:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1645.0>:ns_port_sup:restart_port:92] restarting port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} [ns_server:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: EOL on stdin. Exiting [menelaus:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.397.0>:menelaus_web:handle_streaming:707] menelaus_web streaming socket closed by client [ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1647.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000 [menelaus:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1568.0>:menelaus_web_buckets:handle_bucket_delete:192] Deleted bucket "default" [error_logger:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.1647.0>}, {name, {moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1620.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61725 path /pools/default/saslBucketsStreaming [ns_server:info] [2012-05-14 13:45:18] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: 2012-05-14 13:45:17: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.1648.0>: 2012-05-14 13:45:17: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) [menelaus:info] [2012-05-14 13:45:57] [ns_1@10.1.3.83:<0.1736.0>:menelaus_web_buckets:do_bucket_create:262] Created bucket "default" of type: membase [{num_replicas,1},{ram_quota,1048576000},{auth_type,sasl}] [ns_server:info] [2012-05-14 13:45:57] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[[{map,[]}, {fastForwardMap,[]}, {sasl_password,[]}, {num_replicas,1}, {ram_quota,1048576000}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,[]}]]}] [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw" [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:84] Starting new child: {{per_bucket_sup,"default"}, {single_bucket_sup,start_link,["default"]}, permanent,infinity,supervisor, [single_bucket_sup]} [error_logger:info] [2012-05-14 13:45:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1768.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[{"default", [{map,[]}, {fastForwardMap,[]}, {sasl_password,[]}, {num_replicas,1}, {ram_quota,1048576000}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.3.83']}]}]}] [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [stats:error] [2012-05-14 13:45:57] [ns_1@10.1.3.83:<0.1736.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:info] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: 2012-05-14 13:45:59: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.1648.0>: "name": "default", moxi<0.1648.0>: "nodeLocator": "vbucket", moxi<0.1648.0>: "saslPassword": "", moxi<0.1648.0>: "nodes": [{ moxi<0.1648.0>: "replication": "undefined", moxi<0.1648.0>: "clusterMembership": "active", moxi<0.1648.0>: "status": "warmup", moxi<0.1648.0>: "hostname": "10.1.3.83:8091", moxi<0.1648.0>: "clusterCompatibility": 1, moxi<0.1648.0>: "version": "1.8.1-815-rel-enterprise", moxi<0.1648.0>: "os": "windows", moxi<0.1648.0>: "ports": { moxi<0.1648.0>: "proxy": 11211, moxi<0.1648.0>: "direct": 11210 moxi<0.1648.0>: } moxi<0.1648.0>: }], moxi<0.1648.0>: "vBucketServerMap": { moxi<0.1648.0>: "hashAlgorithm": "CRC", moxi<0.1648.0>: "numReplicas": 1, moxi<0.1648.0>: "serverList": ["10.1.3.83:11210"], moxi<0.1648.0>: "vBucketMap": [] moxi<0.1648.0>: } moxi<0.1648.0>: }) [stats:error] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1671.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:45:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:45:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1611.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:45:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1791.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:00] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:46:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:46:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1791.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1801.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:46:03] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:46:03] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1801.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1812.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:06] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:46:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1765.0>} [ns_server:error] [2012-05-14 13:46:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:info] [2012-05-14 13:46:07] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: 2012-05-14 13:46:09: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.1648.0>: "name": "default", moxi<0.1648.0>: "nodeLocator": "vbucket", moxi<0.1648.0>: "saslPassword": "", moxi<0.1648.0>: "nodes": [{ moxi<0.1648.0>: "replication": 1, moxi<0.1648.0>: "clusterMembership": "active", moxi<0.1648.0>: "status": "warmup", moxi<0.1648.0>: "hostname": "10.1.3.83:8091", moxi<0.1648.0>: "clusterCompatibility": 1, moxi<0.1648.0>: "version": "1.8.1-815-rel-enterprise", moxi<0.1648.0>: "os": "windows", moxi<0.1648.0>: "ports": { moxi<0.1648.0>: "proxy": 11211, moxi<0.1648.0>: "direct": 11210 moxi<0.1648.0>: } moxi<0.1648.0>: }], moxi<0.1648.0>: "vBucketServerMap": { moxi<0.1648.0>: "hashAlgorithm": "CRC", moxi<0.1648.0>: "numReplicas": 1, moxi<0.1648.0>: "serverList": ["10.1.3.83:11210"], moxi<0.1648.0>: "vBucketMap": [] moxi<0.1648.0>: } moxi<0.1648.0>: }) [error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1770.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1768.0>] messages: [check_started,check_started, {'$gen_call',{<0.1787.0>, {#Ref<0.0.0.20455>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.1797.0>,#Ref<0.0.0.20622>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.20680>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.1808.0>, {#Ref<0.0.0.20820>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.21031>},connected}, check_started] links: [<0.64.0>,<0.1769.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7355 neighbours: [error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1768.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1768.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1823.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1584.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1824.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1812.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1830.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:46:09] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1830.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1841.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:12] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:46:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:46:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:46:14] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:46:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1841.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1850.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:46:15] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:46:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1765.0>} [ns_doctor:debug] [2012-05-14 13:46:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28368,20008}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13761960}, {processes,5346264}, {processes_used,5336072}, {system,8415696}, {atom,702357}, {atom_used,681123}, {binary,298024}, {code,5596508}, {ets,514300}]}, {system_stats, [{cpu_utilization_rate,27.418126428027417}, {swap_total,9850605568}, {swap_used,770727936}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,533}, {memory_data,{4294500352,650952704,{<0.1280.0>,300780}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3640164352}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{532672,0}}, {context_switches,{78096,0}}, {garbage_collection,{13420,56277027,0}}, {io,{{input,9277932},{output,5261376}}}, {reductions,{18692868,548379}}, {run_queue,0}, {runtime,{16953,953}}]}]}] [ns_server:error] [2012-05-14 13:46:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:46:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1850.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1880.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:<0.1736.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:18] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1828.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1823.0>] messages: [check_started,check_started, {'$gen_call',{<0.1831.0>, {#Ref<0.0.0.21154>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.21428>},connected}, check_started, {'$gen_call',{<0.1846.0>,#Ref<0.0.0.21431>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.1851.0>, {#Ref<0.0.0.21491>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.21759>},connected}, check_started] links: [<0.64.0>,<0.1827.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.1765.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4288 neighbours: [error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1824.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1884.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1823.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1823.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1885.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [stats:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1880.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1899.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:46:21] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:46:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:46:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1899.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1910.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:24] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1910.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1918.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:46:27] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:46:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:info] [2012-05-14 13:46:28] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1887.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1885.0>] messages: [check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.22179>},connected}, check_started,check_started, {'$gen_call',{<0.1906.0>,#Ref<0.0.0.22185>},topkeys}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.1927.0>, {#Ref<0.0.0.22481>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.22511>},connected}, check_started,check_started] links: [<0.64.0>,<0.1886.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7347 neighbours: [error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1885.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1930.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1885.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1884.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1933.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1918.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1940.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:30] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:46:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:46:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1940.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1948.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:46:33] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:46:34] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1948.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1960.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:36] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:46:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1925.0>} [ns_server:error] [2012-05-14 13:46:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:46:38] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1930.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.1971.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1932.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1930.0>] messages: [check_started,check_started, {'$gen_call',{<0.1936.0>, {#Ref<0.0.0.22589>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.22778>},connected}, check_started,check_started, {'$gen_call',{<0.1949.0>,#Ref<0.0.0.22803>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.1956.0>, {#Ref<0.0.0.22927>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.23118>},connected}, check_started,check_started] links: [<0.64.0>,<0.1931.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1930.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1933.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1974.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1960.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1978.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:39] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:46:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1978.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1989.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:42] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:46:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:46:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:46:44] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:46:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1989.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.1998.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:45] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:46:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.1925.0>} [ns_server:warn] [2012-05-14 13:46:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:46:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.1998.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2011.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:48] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.1973.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.1971.0>] messages: [check_started,check_started, {'$gen_call',{<0.1982.0>, {#Ref<0.0.0.23263>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.23478>},connected}, check_started,check_started, {'$gen_call',{<0.1994.0>,#Ref<0.0.0.23489>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.1999.0>, {#Ref<0.0.0.23545>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.23718>},connected}, check_started,check_started] links: [<0.64.0>,<0.1972.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.1971.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2016.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.1974.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1971.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.1925.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4286 neighbours: [error_logger:info] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2017.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2011.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2027.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:51] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:46:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:46:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2027.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2038.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:46:54] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:46:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2038.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:46:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2046.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:46:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:46:57] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:46:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:46:58] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2019.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2016.0>] messages: [check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.24078>},connected}, check_started,check_started, {'$gen_call',{<0.2036.0>,#Ref<0.0.0.24100>},topkeys}, check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.2051.0>, {#Ref<0.0.0.24299>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.24326>},connected}, check_started,check_started] links: [<0.64.0>,<0.2018.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7347 neighbours: [error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2016.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2058.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2017.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2016.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2059.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:46:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2046.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2068.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:00] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:47:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:47:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2068.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2076.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:03] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:04] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2076.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2088.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:06] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:47:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2049.0>} [ns_server:error] [2012-05-14 13:47:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:47:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:08] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2061.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2058.0>] messages: [check_started,check_started, {'$gen_call',{<0.2066.0>, {#Ref<0.0.0.24508>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.24679>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.2083.0>,#Ref<0.0.0.24799>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2086.0>, {#Ref<0.0.0.24854>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.25018>},connected}, check_started,check_started,check_started, {'EXIT',<0.2060.0>,shutdown}] links: [<0.64.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7355 neighbours: [error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2058.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2099.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2058.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2059.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2101.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2088.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2106.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:09] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:47:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2106.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2117.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:47:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:47:12] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:15] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:47:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2117.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2126.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:15] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:16] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2049.0>} [ns_doctor:debug] [2012-05-14 13:47:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28428,973010}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13375976}, {processes,4945240}, {processes_used,4933480}, {system,8430736}, {atom,702357}, {atom_used,681123}, {binary,296360}, {code,5596508}, {ets,531324}]}, {system_stats, [{cpu_utilization_rate,42.60369815092454}, {swap_total,9850605568}, {swap_used,728403968}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,593}, {memory_data,{4294500352,654356480,{<0.1620.0>,415388}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3683676160}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{592672,0}}, {context_switches,{87884,0}}, {garbage_collection,{14613,64538730,0}}, {io,{{input,9535379},{output,5805069}}}, {reductions,{20013748,362705}}, {run_queue,0}, {runtime,{19718,843}}]}]}] [ns_server:error] [2012-05-14 13:47:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:47:18] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2126.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2141.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:18] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:19] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2102.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2099.0>] messages: [check_started,check_started, {'$gen_call',{<0.2113.0>, {#Ref<0.0.0.25207>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.25296>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.2124.0>,#Ref<0.0.0.25404>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2133.0>, {#Ref<0.0.0.25546>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.25622>},connected}, check_started,check_started,check_started] links: [<0.64.0>,<0.2100.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.2049.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4288 neighbours: [error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2099.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2099.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2146.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2101.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2148.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:20] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2141.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2157.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:21] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:22] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:47:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:47:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2157.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2168.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:24] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:47:24] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:26] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2168.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2176.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:47:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:47:27] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:28] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:29] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2149.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2146.0>] messages: [check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.26005>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.2166.0>,#Ref<0.0.0.26036>},topkeys}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.2181.0>, {#Ref<0.0.0.26232>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.26255>},connected}, check_started,check_started,check_started] links: [<0.64.0>,<0.2147.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7347 neighbours: [error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2146.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2188.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2148.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2190.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2146.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:47:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2176.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2197.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:30] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:47:31] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:47:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:47:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2197.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2206.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:34] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:35] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:47:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2206.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2217.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:47:37] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:47:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2179.0>} [ns_server:error] [2012-05-14 13:47:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:47:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2217.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2230.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2191.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2188.0>] messages: [check_started,check_started, {'$gen_call',{<0.2198.0>, {#Ref<0.0.0.26453>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.26606>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2213.0>,#Ref<0.0.0.26726>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2218.0>, {#Ref<0.0.0.26791>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.26937>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.2189.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2190.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2231.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2188.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2188.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2232.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [stats:error] [2012-05-14 13:47:40] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2230.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2247.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:47:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:47:43] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2247.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2256.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:47:45] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:47:46] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2179.0>} [ns_server:error] [2012-05-14 13:47:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:47:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2256.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2269.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:49] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2234.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2232.0>] messages: [check_started,check_started, {'$gen_call',{<0.2243.0>, {#Ref<0.0.0.27132>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.27220>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2254.0>,#Ref<0.0.0.27331>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2263.0>, {#Ref<0.0.0.27469>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.27533>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.2233.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2232.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.2179.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4286 neighbours: [error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2231.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2232.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2277.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2276.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2269.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2285.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:47:52] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:47:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:47:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2285.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2295.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:47:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:47:55] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:47:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2295.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2304.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:47:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:47:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:47:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:47:59] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2279.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2277.0>] messages: [check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.27899>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.2296.0>,#Ref<0.0.0.27963>},topkeys}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.2309.0>, {#Ref<0.0.0.28128>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.28155>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.2278.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7349 neighbours: [error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2277.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2318.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2276.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2321.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2277.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:48:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2304.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2325.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:01] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:48:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:48:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2325.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2334.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:48:04] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:48:05] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2334.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2345.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:07] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:48:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2307.0>} [ns_server:error] [2012-05-14 13:48:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:48:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2345.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2358.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:48:10] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2320.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2318.0>] messages: [check_started,check_started, {'$gen_call',{<0.2326.0>, {#Ref<0.0.0.28363>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.28499>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.2341.0>,#Ref<0.0.0.28637>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2346.0>, {#Ref<0.0.0.28705>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.28841>},connected}, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.2319.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7357 neighbours: [error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2321.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2318.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2318.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2362.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2361.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2358.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2375.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:48:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:48:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:13] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2375.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2384.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:48:16] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:48:16] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:48:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2307.0>} [ns_doctor:debug] [2012-05-14 13:48:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28490,20010}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13265808}, {processes,4880500}, {processes_used,4868220}, {system,8385308}, {atom,702357}, {atom_used,681123}, {binary,264088}, {code,5596508}, {ets,517852}]}, {system_stats, [{cpu_utilization_rate,27.768057985503624}, {swap_total,9850605568}, {swap_used,719425536}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,655}, {memory_data,{4294500352,602001408,{<0.1620.0>,415388}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3693133824}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{652672,0}}, {context_switches,{97160,0}}, {garbage_collection,{15745,72376666,0}}, {io,{{input,9545962},{output,6119630}}}, {reductions,{21272892,369478}}, {run_queue,0}, {runtime,{22687,844}}]}]}] [ns_server:error] [2012-05-14 13:48:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:48:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2384.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2399.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:19] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:20] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2364.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2362.0>] messages: [check_started,check_started, {'$gen_call',{<0.2371.0>, {#Ref<0.0.0.29045>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.29116>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.2382.0>,#Ref<0.0.0.29247>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2392.0>, {#Ref<0.0.0.29395>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.29445>},connected}, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.2363.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2362.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2406.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.2307.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4287 neighbours: [error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2361.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2362.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2408.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2399.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2415.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:48:22] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:48:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:48:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2415.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2425.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:25] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2425.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2434.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:48:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:48:28] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2409.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2406.0>] messages: [check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.29812>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.2426.0>,#Ref<0.0.0.29877>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2439.0>, {#Ref<0.0.0.30038>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.30069>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2407.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7349 neighbours: [error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2406.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2406.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2448.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2408.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2451.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2434.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2455.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:31] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:48:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:48:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2455.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2464.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:48:34] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:48:36] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2464.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2475.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:48:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2437.0>} [ns_server:error] [2012-05-14 13:48:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:48:37] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2475.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2488.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:48:40] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2450.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2448.0>] messages: [check_started,check_started, {'$gen_call',{<0.2456.0>, {#Ref<0.0.0.30288>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.30417>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.2471.0>,#Ref<0.0.0.30556>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2476.0>, {#Ref<0.0.0.30619>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.30671>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2449.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2448.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2448.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2491.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2451.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2493.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2488.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2505.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:48:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:48:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:43] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2505.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2514.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:48:46] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:48:46] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:48:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2437.0>} [ns_server:error] [2012-05-14 13:48:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:48:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2514.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2527.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:49] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:50] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2494.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2491.0>] messages: [check_started,check_started, {'$gen_call',{<0.2501.0>, {#Ref<0.0.0.30967>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.31038>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.2512.0>,#Ref<0.0.0.31167>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2521.0>, {#Ref<0.0.0.31309>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.31354>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2492.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2491.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2534.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.2437.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4286 neighbours: [error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2493.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2536.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2491.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:48:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2527.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2543.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:48:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:48:52] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:48:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2543.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2553.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:48:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:55] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:48:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2553.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:48:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2565.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:48:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:48:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:48:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:48:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2565.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2577.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2537.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2534.0>] messages: [check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.31636>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.2556.0>,#Ref<0.0.0.31803>},topkeys}, check_started,check_started,check_started, {'$gen_call',{<0.2567.0>, {#Ref<0.0.0.31933>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.31967>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2535.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7349 neighbours: [error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2534.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2534.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2578.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2536.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2580.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:01] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:49:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:49:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2577.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2594.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:05] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2594.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2603.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:49:06] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:49:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2564.0>} [ns_server:error] [2012-05-14 13:49:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:49:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:08] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2603.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2618.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:49:10] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2578.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2619.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2578.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2583.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2578.0>] messages: [check_started,check_started, {'$gen_call',{<0.2586.0>, {#Ref<0.0.0.32198>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.32314>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.2601.0>,#Ref<0.0.0.32479>},topkeys}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.32573>},connected}, check_started, {'$gen_call',{<0.2610.0>, {#Ref<0.0.0.32574>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2579.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7358 neighbours: [error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2580.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2622.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:49:11] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2618.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2633.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:49:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:49:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:14] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2633.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2643.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:49:16] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:49:17] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:49:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2564.0>} [ns_doctor:debug] [2012-05-14 13:49:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28550,927009}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13424632}, {processes,4987580}, {processes_used,4975268}, {system,8437052}, {atom,702357}, {atom_used,681123}, {binary,299792}, {code,5596508}, {ets,533852}]}, {system_stats, [{cpu_utilization_rate,26.979765176117912}, {swap_total,9850605568}, {swap_used,720330752}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,715}, {memory_data,{4294500352,601063424,{<0.1620.0>,415388}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3694972928}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{712672,0}}, {context_switches,{106253,0}}, {garbage_collection,{16890,79899708,0}}, {io,{{input,9556449},{output,6430954}}}, {reductions,{22509285,340449}}, {run_queue,0}, {runtime,{25500,688}}]}]}] [ns_server:error] [2012-05-14 13:49:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:49:18] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2643.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2670.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:20] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:20] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:21] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2621.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2619.0>] messages: [check_started,check_started, {'$gen_call',{<0.2631.0>, {#Ref<0.0.0.32897>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.32937>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.2644.0>,#Ref<0.0.0.33096>},topkeys}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.33347>},connected}, check_started, {'$gen_call',{<0.2668.0>, {#Ref<0.0.0.33372>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2620.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7353 neighbours: [error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.2564.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4288 neighbours: [error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2622.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2677.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2619.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2678.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2619.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2670.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2688.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:22] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:49:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:49:23] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:24] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2688.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2696.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:26] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:26] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2696.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2710.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:49:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:49:28] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:29] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:30] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2710.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2720.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:49:31] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2680.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2678.0>] messages: [check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.33653>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.2699.0>,#Ref<0.0.0.33824>},topkeys}, check_started,check_started,check_started, {'$gen_call',{<0.2708.0>, {#Ref<0.0.0.33951>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.33987>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started] links: [<0.64.0>,<0.2679.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7350 neighbours: [error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2678.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2721.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2677.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2723.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2678.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [ns_server:warn] [2012-05-14 13:49:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:32] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:49:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:49:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2720.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2737.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:35] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2737.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2746.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:49:37] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:49:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2706.0>} [ns_server:error] [2012-05-14 13:49:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:49:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:38] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2746.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2761.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:49:41] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2724.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2721.0>] messages: [check_started,check_started, {'$gen_call',{<0.2729.0>, {#Ref<0.0.0.34229>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.34344>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2744.0>,#Ref<0.0.0.34506>},topkeys}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.34589>},connected}, check_started,check_started, {'$gen_call',{<0.2753.0>, {#Ref<0.0.0.34595>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2722.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7353 neighbours: [error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2721.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2721.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2765.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2723.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2767.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:49:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:49:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2761.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2776.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:49:44] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2776.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2786.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:49:47] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:49:47] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:49:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2706.0>} [ns_server:error] [2012-05-14 13:49:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:49:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2786.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2798.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:49:50] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:49:51] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2768.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2765.0>] messages: [check_started,check_started, {'$gen_call',{<0.2774.0>, {#Ref<0.0.0.34915>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.34945>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2787.0>,#Ref<0.0.0.35121>},topkeys}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.35265>},connected}, check_started,check_started, {'$gen_call',{<0.2796.0>, {#Ref<0.0.0.35275>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2766.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7353 neighbours: [error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2765.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.2706.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4286 neighbours: [error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2765.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2805.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2767.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2806.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2798.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2816.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:<0.1744.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:49:53] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:49:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:49:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2816.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2836.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:49:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:49:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2836.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:49:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2848.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:49:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:50:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2848.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2852.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:50:01] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2805.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2855.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2805.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2808.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2805.0>] messages: [check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.35734>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2839.0>,#Ref<0.0.0.35883>},topkeys}, check_started,check_started, {'$gen_call',{<0.2844.0>, {#Ref<0.0.0.35917>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.35948>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.2807.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2806.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2858.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:50:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:50:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2852.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2865.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:50:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2865.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2870.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:50:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2842.0>} [ns_server:error] [2012-05-14 13:50:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:info] [2012-05-14 13:50:07] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:50:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2870.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2881.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:50:11] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2857.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2855.0>] messages: [check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.36139>},connected}, check_started, {'$gen_call',{<0.2861.0>, {#Ref<0.0.0.36144>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2868.0>,#Ref<0.0.0.36251>},topkeys}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.36315>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.2877.0>, {#Ref<0.0.0.36336>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2856.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7357 neighbours: [error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2855.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2855.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2884.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2858.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2887.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:50:12] [ns_1@10.1.3.83:<0.1744.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:50:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:50:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2881.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2895.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:50:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2895.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2902.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:16] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:50:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2842.0>} [ns_doctor:debug] [2012-05-14 13:50:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28611,864006}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13713416}, {processes,5254576}, {processes_used,5243072}, {system,8458840}, {atom,702357}, {atom_used,681123}, {binary,305696}, {code,5596508}, {ets,549180}]}, {system_stats, [{cpu_utilization_rate,27.71385692846423}, {swap_total,9850605568}, {swap_used,720048128}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,776}, {memory_data,{4294500352,599257088,{<0.1620.0>,415388}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3696807936}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{772672,0}}, {context_switches,{115513,0}}, {garbage_collection,{18099,87193107,0}}, {io,{{input,9812485},{output,6869119}}}, {reductions,{23812751,291826}}, {run_queue,0}, {runtime,{28546,734}}]}]}] [ns_server:error] [2012-05-14 13:50:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:info] [2012-05-14 13:50:17] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:50:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2902.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2911.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:50:19] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2818.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2927.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37061>} event handler: ok [ns_server:debug] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2930.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37074>} event handler: ok [menelaus:warn] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2818.0>:menelaus_web:loop:334] Server error during processing: ["web request failed", {path, "/pools/default/buckets/default/nodes/10.1.3.83:8091/stats"}, {type,error}, {what,function_clause}, {trace, [{lists,map, [#Fun, undefined]}, {menelaus_stats,jsonify_hks,1}, {menelaus_stats, '-handle_bucket_node_stats/4-fun-0-',7}, {menelaus_web_buckets, checking_bucket_access,4}, {menelaus_web, handle_abortable_get_request,2}, {menelaus_web_buckets, checking_bucket_access,4}, {menelaus_web,loop,3}, {mochiweb_http,headers,5}]}] [ns_server:warn] [2012-05-14 13:50:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2911.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2936.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:22] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2886.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2884.0>] messages: [check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.36582>},connected}, check_started, {'$gen_call',{<0.2896.0>, {#Ref<0.0.0.36596>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.2904.0>,#Ref<0.0.0.36701>},topkeys}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.36752>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.2912.0>, {#Ref<0.0.0.36803>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2885.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7353 neighbours: [error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2884.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2937.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.2842.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4288 neighbours: [error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2887.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2940.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2884.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [stats:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:50:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2936.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2951.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:debug] [2012-05-14 13:50:25] [ns_1@10.1.3.83:<0.2956.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37399>} event handler: ok [ns_server:debug] [2012-05-14 13:50:25] [ns_1@10.1.3.83:<0.2958.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37408>} event handler: ok [menelaus:warn] [2012-05-14 13:50:25] [ns_1@10.1.3.83:<0.2921.0>:menelaus_web:loop:334] Server error during processing: ["web request failed", {path, "/pools/default/buckets/default/nodes/10.1.3.83:8091/stats"}, {type,error}, {what,function_clause}, {trace, [{lists,map, [#Fun, undefined]}, {menelaus_stats,jsonify_hks,1}, {menelaus_stats, '-handle_bucket_node_stats/4-fun-0-',7}, {menelaus_web_buckets, checking_bucket_access,4}, {menelaus_web, handle_abortable_get_request,2}, {menelaus_web_buckets, checking_bucket_access,4}, {menelaus_web,loop,3}, {mochiweb_http,headers,5}]}] [ns_server:info] [2012-05-14 13:50:25] [ns_1@10.1.3.83:ns_log:ns_log:handle_cast:115] suppressing duplicate log menelaus_web:19("Server error during processing: [\"web request failed\",\n {path,\n \"/pools/default/buckets/default/nodes/10.1.3.83:8091/stats\"},\n {type,error},\n {what,function_clause},\n {trace,\n [{lists,map,\n [#Fun,\n undefined]},\n {menelaus_stats,jsonify_hks,1},\n {menelaus_stats,\n '-handle_bucket_node_stats/4-fun-0-',7},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,\n handle_abortable_get_request,2},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,loop,3},\n {mochiweb_http,headers,5}]}]") because it's been seen 1 times in the past 5.016006 secs (last seen 5.016006 secs ago [ns_server:warn] [2012-05-14 13:50:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:50:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:50:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2951.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2970.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2970.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2974.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:<0.1744.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:50:32] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2939.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2937.0>] messages: [check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.37302>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.2961.0>,#Ref<0.0.0.37454>},topkeys}, check_started, {'$gen_call',{<0.2966.0>, {#Ref<0.0.0.37481>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.37509>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.2938.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7350 neighbours: [error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2937.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.2980.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2940.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2937.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2982.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:50:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2974.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.2990.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:50:33] [ns_1@10.1.3.83:<0.2993.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37814>} event handler: ok [ns_server:debug] [2012-05-14 13:50:33] [ns_1@10.1.3.83:<0.2995.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37823>} event handler: ok [menelaus:warn] [2012-05-14 13:50:33] [ns_1@10.1.3.83:<0.2954.0>:menelaus_web:loop:334] Server error during processing: ["web request failed", {path, "/pools/default/buckets/default/nodes/10.1.3.83:8091/stats"}, {type,error}, {what,function_clause}, {trace, [{lists,map, [#Fun, undefined]}, {menelaus_stats,jsonify_hks,1}, {menelaus_stats, '-handle_bucket_node_stats/4-fun-0-',7}, {menelaus_web_buckets, checking_bucket_access,4}, {menelaus_web, handle_abortable_get_request,2}, {menelaus_web_buckets, checking_bucket_access,4}, {menelaus_web,loop,3}, {mochiweb_http,headers,5}]}] [ns_server:info] [2012-05-14 13:50:33] [ns_1@10.1.3.83:ns_log:ns_log:handle_cast:115] suppressing duplicate log menelaus_web:19("Server error during processing: [\"web request failed\",\n {path,\n \"/pools/default/buckets/default/nodes/10.1.3.83:8091/stats\"},\n {type,error},\n {what,function_clause},\n {trace,\n [{lists,map,\n [#Fun,\n undefined]},\n {menelaus_stats,jsonify_hks,1},\n {menelaus_stats,\n '-handle_bucket_node_stats/4-fun-0-',7},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,\n handle_abortable_get_request,2},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,loop,3},\n {mochiweb_http,headers,5}]}]") because it's been seen 2 times in the past 13.110007 secs (last seen 8.094001 secs ago [ns_server:warn] [2012-05-14 13:50:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.2990.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3002.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:50:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2964.0>} [ns_server:error] [2012-05-14 13:50:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:info] [2012-05-14 13:50:38] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:50:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3002.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3014.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:50:40] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:50:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.2983.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.2980.0>] messages: [{'$gen_call',{<0.231.0>,#Ref<0.0.0.37755>},connected}, check_started,check_started, {'$gen_call',{<0.2986.0>, {#Ref<0.0.0.37760>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3003.0>,#Ref<0.0.0.37932>},topkeys}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.37978>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3010.0>, {#Ref<0.0.0.38002>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.2981.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7353 neighbours: [error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2980.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.2980.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3021.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.2982.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3024.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3014.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3028.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:50:45] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:50:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3028.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3038.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:50:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.2964.0>} [ns_server:error] [2012-05-14 13:50:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3038.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3054.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:50:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:50:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:50:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3054.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3061.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3021.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3021.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3062.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3023.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3021.0>] messages: [{'$gen_call',{<0.231.0>,#Ref<0.0.0.38254>},connected}, check_started,check_started, {'$gen_call',{<0.3029.0>, {#Ref<0.0.0.38278>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3039.0>,#Ref<0.0.0.38410>},topkeys}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.38456>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3055.0>, {#Ref<0.0.0.38589>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.3022.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7353 neighbours: [error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3024.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3065.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:50:53] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason shutdown [ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:91] Stopping child for dead bucket: {{per_bucket_sup,"default"}, <0.3062.0>,supervisor, [single_bucket_sup]} [ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[]}] [ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw" [ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:warn] [2012-05-14 13:50:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:50:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:50:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:50:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:51:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: shutdown_error Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3062.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3074.0>:ns_pubsub:do_subscribe_link:132] Parent process exited with reason normal [ns_server:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:idle:373] Restarting moxi on nodes ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3074.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.38838>} event handler: ok [error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3064.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3062.0>] messages: [{'$gen_call',{<0.2964.0>,#Ref<0.0.0.38676>}, list_vbuckets_prevstate}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.38748>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.39057>},connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started,check_started, check_started,check_started, {'EXIT',<0.3063.0>,shutdown}] links: [<0.64.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7338 neighbours: [error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3062.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [ns_server:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3102.0>:ns_port_sup:restart_port:92] restarting port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env,[{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]} [ns_server:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: EOL on stdin. Exiting [menelaus:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.1620.0>:menelaus_web:handle_streaming:707] menelaus_web streaming socket closed by client [ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3104.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000 [menelaus:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.2920.0>:menelaus_web_buckets:handle_bucket_delete:192] Deleted bucket "default" [error_logger:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_port_sup} started: [{pid,<0.3104.0>}, {name, {moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]}}, {mfargs, {supervisor_cushion,start_link, [moxi,5000,ns_port_server,start_link, [moxi, "c:/Program Files/Couchbase/Server/bin/moxi", ["-Z", "port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200", "-z", "url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming", "-p","0","-Y","y","-O","stderr",[]], [{env, [{"EVENT_NOSELECT","1"}, {"MOXI_SASL_PLAIN_USR","Administrator"}, {"MOXI_SASL_PLAIN_PWD","password"}]}, use_stdio,exit_status,port_server_send_eol, stderr_to_stdout,stream]]]}}, {restart_type,permanent}, {shutdown,10000}, {child_type,worker}] [ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3075.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61932 path /pools/default/saslBucketsStreaming [ns_server:info] [2012-05-14 13:51:03] [ns_1@10.1.3.83:<0.3105.0>:ns_port_server:log:161] moxi<0.3105.0>: 2012-05-14 13:51:02: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13) moxi<0.3105.0>: 2012-05-14 13:51:02: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8) [ns_doctor:debug] [2012-05-14 13:51:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28672,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,13993296}, {processes,5227196}, {processes_used,5213708}, {system,8766100}, {atom,728565}, {atom_used,713266}, {binary,448024}, {code,5718270}, {ets,565524}]}, {system_stats, [{cpu_utilization_rate,25.4}, {swap_total,9850605568}, {swap_used,718110720}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,837}, {memory_data,{4294500352,597471232,{<0.1620.0>,415388}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3698405376}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{827688,0}}, {context_switches,{122991,0}}, {garbage_collection,{19152,92275520,0}}, {io,{{input,9961816},{output,7145454}}}, {reductions,{24890443,196517}}, {run_queue,0}, {runtime,{31093,453}}]}]}] [ns_doctor:debug] [2012-05-14 13:52:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28732,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,13652992}, {processes,4862152}, {processes_used,4847512}, {system,8790840}, {atom,728565}, {atom_used,713266}, {binary,487136}, {code,5718270}, {ets,551852}]}, {system_stats, [{cpu_utilization_rate,25.418645338665332}, {swap_total,9850605568}, {swap_used,717627392}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,897}, {memory_data,{4294500352,592855040,{<0.2920.0>,300780}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3700641792}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{887656,0}}, {context_switches,{129227,0}}, {garbage_collection,{20095,97210473,0}}, {io,{{input,10001792},{output,7354126}}}, {reductions,{25936110,147304}}, {run_queue,0}, {runtime,{32796,406}}]}]}] [ns_doctor:debug] [2012-05-14 13:53:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28792,645005}}, {active_buckets,[]}, {ready_buckets,[]}, {replication,[]}, {memory, [{total,13901968}, {processes,5154796}, {processes_used,5140652}, {system,8747172}, {atom,728565}, {atom_used,713266}, {binary,426496}, {code,5718270}, {ets,568452}]}, {system_stats, [{cpu_utilization_rate,25.381345336334082}, {swap_total,9850605568}, {swap_used,723779584}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,957}, {memory_data,{4294500352,593780736,{<0.3152.0>,300780}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3688812544}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{947656,0}}, {context_switches,{135133,0}}, {garbage_collection,{21102,101126711,0}}, {io,{{input,10285014},{output,7756644}}}, {reductions,{26936431,147602}}, {run_queue,0}, {runtime,{34265,328}}]}]}] [menelaus:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3318.0>:menelaus_web_buckets:do_bucket_create:262] Created bucket "default" of type: membase [{num_replicas,1},{ram_quota,3435134976},{auth_type,sasl}] [ns_server:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[[{map,[]}, {fastForwardMap,[]}, {sasl_password,[]}, {num_replicas,1}, {ram_quota,3435134976}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,[]}]]}] [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw" [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:84] Starting new child: {{per_bucket_sup,"default"}, {single_bucket_sup,start_link,["default"]}, permanent,infinity,supervisor, [single_bucket_sup]} [error_logger:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3488.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change: buckets -> [{configs,[{"default", [{map,[]}, {fastForwardMap,[]}, {sasl_password,[]}, {num_replicas,1}, {ram_quota,3435134976}, {auth_type,sasl}, {type,membase}, {num_vbuckets,1024}, {servers,['ns_1@10.1.3.83']}]}]}] [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config [ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done [stats:error] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3318.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3105.0>:ns_port_server:log:161] moxi<0.3105.0>: 2012-05-14 13:53:30: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.3105.0>: "name": "default", moxi<0.3105.0>: "nodeLocator": "vbucket", moxi<0.3105.0>: "saslPassword": "", moxi<0.3105.0>: "nodes": [{ moxi<0.3105.0>: "replication": "undefined", moxi<0.3105.0>: "clusterMembership": "active", moxi<0.3105.0>: "status": "warmup", moxi<0.3105.0>: "hostname": "10.1.3.83:8091", moxi<0.3105.0>: "clusterCompatibility": 1, moxi<0.3105.0>: "version": "1.8.1-815-rel-enterprise", moxi<0.3105.0>: "os": "windows", moxi<0.3105.0>: "ports": { moxi<0.3105.0>: "proxy": 11211, moxi<0.3105.0>: "direct": 11210 moxi<0.3105.0>: } moxi<0.3105.0>: }], moxi<0.3105.0>: "vBucketServerMap": { moxi<0.3105.0>: "hashAlgorithm": "CRC", moxi<0.3105.0>: "numReplicas": 1, moxi<0.3105.0>: "serverList": ["10.1.3.83:11210"], moxi<0.3105.0>: "vBucketMap": [] moxi<0.3105.0>: } moxi<0.3105.0>: }) [stats:error] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3313.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3313.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:53:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:53:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3061.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3509.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:53:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:53:31] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:53:31] [ns_1@10.1.3.83:<0.3313.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:error] [2012-05-14 13:53:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:53:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:53:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3509.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3529.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:53:34] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:53:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:53:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3529.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3534.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:53:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:53:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3486.0>} [ns_server:error] [2012-05-14 13:53:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:info] [2012-05-14 13:53:37] [ns_1@10.1.3.83:<0.3105.0>:ns_port_server:log:161] moxi<0.3105.0>: 2012-05-14 13:53:39: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({ moxi<0.3105.0>: "name": "default", moxi<0.3105.0>: "nodeLocator": "vbucket", moxi<0.3105.0>: "saslPassword": "", moxi<0.3105.0>: "nodes": [{ moxi<0.3105.0>: "replication": 1, moxi<0.3105.0>: "clusterMembership": "active", moxi<0.3105.0>: "status": "warmup", moxi<0.3105.0>: "hostname": "10.1.3.83:8091", moxi<0.3105.0>: "clusterCompatibility": 1, moxi<0.3105.0>: "version": "1.8.1-815-rel-enterprise", moxi<0.3105.0>: "os": "windows", moxi<0.3105.0>: "ports": { moxi<0.3105.0>: "proxy": 11211, moxi<0.3105.0>: "direct": 11210 moxi<0.3105.0>: } moxi<0.3105.0>: }], moxi<0.3105.0>: "vBucketServerMap": { moxi<0.3105.0>: "hashAlgorithm": "CRC", moxi<0.3105.0>: "numReplicas": 1, moxi<0.3105.0>: "serverList": ["10.1.3.83:11210"], moxi<0.3105.0>: "vBucketMap": [] moxi<0.3105.0>: } moxi<0.3105.0>: }) [ns_server:info] [2012-05-14 13:53:38] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3490.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3488.0>] messages: [check_started, {'$gen_call',{<0.3505.0>, {#Ref<0.0.0.44696>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.3525.0>,#Ref<0.0.0.45190>},topkeys}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.45209>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.3530.0>, {#Ref<0.0.0.45273>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.45397>},connected}, check_started,check_started] links: [<0.64.0>,<0.3489.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3065.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3541.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3488.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3542.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3488.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [ns_server:warn] [2012-05-14 13:53:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:53:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3534.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3551.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:53:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:53:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:53:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3551.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3555.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:53:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:53:44] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:53:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:53:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3555.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3563.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:53:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3486.0>} [ns_server:warn] [2012-05-14 13:53:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:53:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:<0.3318.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:53:48] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3544.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3542.0>] messages: [check_started,check_started, {'$gen_call',{<0.3547.0>, {#Ref<0.0.0.45513>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.45623>},connected}, check_started,check_started, {'$gen_call',{<0.3556.0>,#Ref<0.0.0.45646>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3559.0>, {#Ref<0.0.0.45689>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.45793>},connected}, check_started,check_started] links: [<0.64.0>,<0.3543.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.3486.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4286 neighbours: [error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3542.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3542.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3571.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3541.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3573.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3563.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3580.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:53:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:53:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:53:51] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:53:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3580.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3590.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:53:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:53:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:53:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3590.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3595.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:53:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:53:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:53:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3595.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3606.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3574.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3571.0>] messages: [check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.46126>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.3593.0>,#Ref<0.0.0.46148>},topkeys}, check_started,check_started,check_started,check_started, check_started,check_started,check_started, {'$gen_call',{<0.3602.0>, {#Ref<0.0.0.46257>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.46281>},connected}, check_started,check_started,check_started] links: [<0.64.0>,<0.3572.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7347 neighbours: [error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3571.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3607.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3571.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3573.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3610.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:53:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3606.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3617.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:54:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:54:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3617.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3624.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:54:04] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:54:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3624.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3630.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:54:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3600.0>} [ns_server:error] [2012-05-14 13:54:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:54:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:54:09] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3609.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3607.0>] messages: [check_started,check_started, {'$gen_call',{<0.3615.0>, {#Ref<0.0.0.46392>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.46477>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.3622.0>,#Ref<0.0.0.46510>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3628.0>, {#Ref<0.0.0.46581>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.46665>},connected}, check_started,check_started,check_started] links: [<0.64.0>,<0.3608.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7355 neighbours: [error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3607.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3607.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3637.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3610.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3640.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3630.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3646.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:54:11] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:54:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:54:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:54:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3646.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3655.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:54:15] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3655.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3662.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:16] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:54:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3600.0>} [ns_doctor:debug] [2012-05-14 13:54:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28849,67009}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,14109000}, {processes,5288192}, {processes_used,5276048}, {system,8820808}, {atom,728565}, {atom_used,713266}, {binary,513080}, {code,5718270}, {ets,554388}]}, {system_stats, [{cpu_utilization_rate,27.411167512690355}, {swap_total,9850605568}, {swap_used,724307968}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,1014}, {memory_data,{4294500352,605745152,{<0.3314.0>,300780}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3688005632}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{1012672,0}}, {context_switches,{143434,0}}, {garbage_collection,{22390,107611232,0}}, {io,{{input,10326587},{output,8138622}}}, {reductions,{28413884,285246}}, {run_queue,0}, {runtime,{36546,812}}]}]}] [ns_server:error] [2012-05-14 13:54:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:54:18] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3662.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3671.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:19] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3639.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3637.0>] messages: [check_started,check_started, {'$gen_call',{<0.3647.0>, {#Ref<0.0.0.46776>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.46931>},connected}, check_started,check_started,check_started, {'$gen_call',{<0.3658.0>,#Ref<0.0.0.46970>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3663.0>, {#Ref<0.0.0.47026>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.47100>},connected}, check_started,check_started,check_started] links: [<0.64.0>,<0.3638.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3637.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3672.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3640.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.3600.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4288 neighbours: [error_logger:info] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3674.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3637.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [ns_server:warn] [2012-05-14 13:54:20] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3671.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3685.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:22] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:54:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:54:24] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3685.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3690.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:26] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:54:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:54:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3690.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3701.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:28] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:54:29] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3672.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3672.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3702.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3675.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3672.0>] messages: [check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.47309>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3688.0>,#Ref<0.0.0.47339>},topkeys}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.3697.0>, {#Ref<0.0.0.47436>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.47464>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.3673.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7347 neighbours: [error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3674.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3705.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:30] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3701.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3712.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:54:31] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:54:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:54:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [stats:error] [2012-05-14 13:54:33] [ns_1@10.1.3.83:<0.3514.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3712.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3727.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:54:35] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:54:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:debug] [2012-05-14 13:54:36] [ns_1@10.1.3.83:<0.3733.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.47976>} event handler: ok [stats:error] [2012-05-14 13:54:36] [ns_1@10.1.3.83:<0.3514.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3727.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3740.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:54:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3695.0>} [ns_server:error] [2012-05-14 13:54:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:54:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:debug] [2012-05-14 13:54:38] [ns_1@10.1.3.83:<0.3747.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.48156>} event handler: ok [error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3702.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3704.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3702.0>] messages: [check_started,check_started, {'$gen_call',{<0.3710.0>, {#Ref<0.0.0.47585>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.47751>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3728.0>,#Ref<0.0.0.47902>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3731.0>, {#Ref<0.0.0.47952>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.48125>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.3703.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3702.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3749.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3705.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3751.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3625.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [stats:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3514.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3740.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3770.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:54:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:54:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:54:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3770.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3775.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:54:45] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3775.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3782.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:54:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3695.0>} [ns_server:error] [2012-05-14 13:54:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:54:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3782.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3789.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:49] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3752.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3749.0>] messages: [check_started,check_started, {'$gen_call',{<0.3771.0>, {#Ref<0.0.0.48452>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.48527>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3778.0>,#Ref<0.0.0.48570>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3783.0>, {#Ref<0.0.0.48627>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.48690>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.3750.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7351 neighbours: [error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3749.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.3695.0> registered_name: [] exception exit: {noproc, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4286 neighbours: [error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3749.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3792.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3751.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3795.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:54:51] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3789.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3806.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:54:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:54:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:54:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:54:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3806.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3811.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:54:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:54:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3811.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3822.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:54:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:54:59] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3794.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3792.0>] messages: [check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.48942>},connected}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3809.0>,#Ref<0.0.0.48995>},topkeys}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.3818.0>, {#Ref<0.0.0.49077>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.49105>},connected}, check_started,check_started,check_started,check_started] links: [<0.64.0>,<0.3793.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7349 neighbours: [error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3792.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3792.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3829.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3795.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3832.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3822.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3837.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:55:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3837.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3843.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:05] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:warn] [2012-05-14 13:55:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3843.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3849.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:55:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3816.0>} [ns_server:error] [2012-05-14 13:55:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:09] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3831.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3829.0>] messages: [check_started,check_started, {'$gen_call',{<0.3835.0>, {#Ref<0.0.0.49315>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.49386>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.3844.0>,#Ref<0.0.0.49443>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3847.0>, {#Ref<0.0.0.49498>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.49567>},connected}, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.3830.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7357 neighbours: [error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3829.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3829.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3858.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3832.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3859.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:55:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3849.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3865.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:55:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3865.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3870.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:15] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:55:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3870.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3877.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3816.0>} [ns_doctor:debug] [2012-05-14 13:55:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28909,927007}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13765264}, {processes,4949160}, {processes_used,4936000}, {system,8816104}, {atom,728565}, {atom_used,713266}, {binary,493592}, {code,5718270}, {ets,569908}]}, {system_stats, [{cpu_utilization_rate,27.375}, {swap_total,9850605568}, {swap_used,725495808}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,1074}, {memory_data,{4294500352,606453760,{<0.3511.0>,300780}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3687280640}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{1072672,0}}, {context_switches,{151010,0}}, {garbage_collection,{23290,112155175,0}}, {io,{{input,10335561},{output,8427342}}}, {reductions,{29434667,265933}}, {run_queue,0}, {runtime,{38921,937}}]}]}] [ns_server:error] [2012-05-14 13:55:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3877.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3899.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [stats:error] [2012-05-14 13:55:19] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3858.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3858.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3906.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3861.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3858.0>] messages: [check_started,check_started, {'$gen_call',{<0.3866.0>, {#Ref<0.0.0.49690>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.49748>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.3873.0>,#Ref<0.0.0.49805>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3879.0>, {#Ref<0.0.0.49868>,'ns_1@10.1.3.83'}}, connected}, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.50010>},connected}, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.3860.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3859.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3910.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3899.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3917.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:55:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:55:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3917.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3922.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:27] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3816.0>} [ns_server:error] [2012-05-14 13:55:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:55:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3922.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3930.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:30] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3906.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3906.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3933.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3908.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3906.0>] messages: [{'$gen_call',{<0.3816.0>,#Ref<0.0.0.50184>}, list_vbuckets_prevstate}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.50311>},connected}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.3920.0>,#Ref<0.0.0.50363>},topkeys}, check_started,check_started,check_started,check_started, check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.50459>},connected}, check_started,check_started,check_started,check_started, check_started] links: [<0.64.0>,<0.3907.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7349 neighbours: [error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.3816.0> registered_name: [] exception exit: {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4291 neighbours: [error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3910.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3936.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3930.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3940.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:55:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3940.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3946.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:55:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3946.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3951.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:55:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:55:39] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3935.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3933.0>] messages: [check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.50659>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.3947.0>,#Ref<0.0.0.50716>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3956.0>, {#Ref<0.0.0.50797>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.50832>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.3934.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7349 neighbours: [error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3933.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3933.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3967.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3936.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3970.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3951.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3974.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:55:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:55:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3974.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3979.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:55:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:46] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:55:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3979.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3986.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3954.0>} [ns_server:error] [2012-05-14 13:55:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:55:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3986.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3993.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:55:50] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3967.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3967.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.3996.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3969.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3967.0>] messages: [check_started,check_started, {'$gen_call',{<0.3975.0>, {#Ref<0.0.0.51040>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.51098>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.3982.0>,#Ref<0.0.0.51165>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.3987.0>, {#Ref<0.0.0.51222>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.51270>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.3968.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3970.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.3999.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.3993.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4008.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:error] [2012-05-14 13:55:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:55:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:55:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4008.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4012.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:info] [2012-05-14 13:55:56] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83'] [ns_server:info] [2012-05-14 13:55:57] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3954.0>} [ns_server:warn] [2012-05-14 13:55:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:55:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:55:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4012.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:55:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4022.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:55:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [stats:error] [2012-05-14 13:55:59] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83'] [error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3996.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.3996.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.4029.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.3998.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.3996.0>] messages: [check_started,check_started, {'$gen_call',{<0.4004.0>, {#Ref<0.0.0.51408>,'ns_1@10.1.3.83'}}, connected}, check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.51471>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.4013.0>,#Ref<0.0.0.51542>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.4018.0>, {#Ref<0.0.0.51588>,'ns_1@10.1.3.83'}}, connected}, {'$gen_call',{<0.231.0>,#Ref<0.0.0.51629>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.3997.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7352 neighbours: [error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.3999.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4032.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:56:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4022.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:56:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4036.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:56:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:56:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:56:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:56:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4036.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:56:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4042.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:56:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:56:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4042.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:56:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4047.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:info] [2012-05-14 13:56:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.3954.0>} [ns_server:error] [2012-05-14 13:56:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [ns_server:warn] [2012-05-14 13:56:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:56:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4047.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4057.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_memcached:init/1 pid: <0.4031.0> registered_name: [] exception exit: couldnt_connect_to_memcached in function gen_server:init_it/6 ancestors: ['single_bucket_sup-default',<0.4029.0>] messages: [{'$gen_call',{<0.3954.0>,#Ref<0.0.0.51788>}, list_vbuckets_prevstate}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.51894>},connected}, check_started,check_started,check_started,check_started, check_started,check_started, {'$gen_call',{<0.4045.0>,#Ref<0.0.0.51970>},topkeys}, check_started,check_started,check_started,check_started, {'$gen_call',{<0.231.0>,#Ref<0.0.0.52062>},connected}, check_started,check_started,check_started,check_started, check_started,check_started] links: [<0.64.0>,<0.4030.0>] dictionary: [] trap_exit: true status: running heap_size: 28657 stack_size: 24 reductions: 7355 neighbours: [error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {couldnt_connect_to_memcached, {gen_server,call,['ns_memcached-default',topkeys,30000]}} Offender: [{pid,<0.4032.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'single_bucket_sup-default'} Context: start_error Reason: couldnt_connect_to_memcached Offender: [{pid,undefined}, {name,{ns_memcached,"default"}}, {mfargs,{ns_memcached,start_link,["default"]}}, {restart_type,permanent}, {shutdown,86400000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:56:10] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default', 'ns_1@10.1.3.83'}, list_vbuckets_prevstate, 30000]}} [error_logger:info] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4058.0>}, {name,hot_keys_keeper}, {mfargs,{hot_keys_keeper,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================CRASH REPORT========================= crasher: initial call: ns_janitor:cleanup/2 pid: <0.3954.0> registered_name: [] exception exit: {couldnt_connect_to_memcached, {gen_server,call, [{'ns_memcached-default','ns_1@10.1.3.83'}, list_vbuckets_prevstate,30000]}} in function gen_server:call/3 in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2 in call from ns_janitor:read_existing_map/4 in call from ns_janitor:do_cleanup/3 ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup, ns_server_cluster_sup,<0.67.0>] messages: [] links: [<0.218.0>] dictionary: [] trap_exit: false status: running heap_size: 28657 stack_size: 24 reductions: 4289 neighbours: [error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,ns_bucket_sup} Context: child_terminated Reason: {{badmatch,{error,shutdown}}, [{single_bucket_sup,'-start_link/1-fun-0-',2}]} Offender: [{pid,<0.4029.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:info] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,ns_bucket_sup} started: [{pid,<0.4059.0>}, {name,{per_bucket_sup,"default"}}, {mfargs,{single_bucket_sup,start_link,["default"]}}, {restart_type,permanent}, {shutdown,infinity}, {child_type,supervisor}] [error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.4029.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]} [ns_server:warn] [2012-05-14 13:56:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:error] [2012-05-14 13:56:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}} [error_logger:error] [2012-05-14 13:56:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4057.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:56:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4067.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_server:warn] [2012-05-14 13:56:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [ns_server:warn] [2012-05-14 13:56:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying. [error_logger:error] [2012-05-14 13:56:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,menelaus_sup} Context: child_terminated Reason: {noproc, {gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute,1}]}} Offender: [{pid,<0.4067.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [error_logger:info] [2012-05-14 13:56:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72] =========================PROGRESS REPORT========================= supervisor: {local,menelaus_sup} started: [{pid,<0.4075.0>}, {name,menelaus_web_alerts_srv}, {mfargs,{menelaus_web_alerts_srv,start_link,[]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,worker}] [ns_doctor:debug] [2012-05-14 13:56:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses: [{'ns_1@10.1.3.83', [{last_heard,{1337,28970,817013}}, {active_buckets,["default"]}, {ready_buckets,[]}, {replication,[{"default",1.0}]}, {memory, [{total,13687056}, {processes,4852820}, {processes_used,4839940}, {system,8834236}, {atom,728565}, {atom_used,713266}, {binary,494184}, {code,5718270}, {ets,587668}]}, {system_stats, [{cpu_utilization_rate,28.125}, {swap_total,9850605568}, {swap_used,727543808}]}, {interesting_stats,[]}, {cluster_compatibility_version,1}, {version, [{ale,"8cffe61"}, {os_mon,"2.2.5"}, {mnesia,"4.4.17"}, {kernel,"2.14.3"}, {sasl,"2.1.9.3"}, {ns_server,"1.8.1-815-rel-enterprise"}, {stdlib,"1.17.3"}]}, {system_arch,"windows"}, {wall_clock,1135}, {memory_data,{4294500352,607203328,{<0.231.0>,300660}}}, {disk_data,[{"C:\\",62810108,36}]}, {system_memory_data, [{total_memory,4294500352}, {free_memory,3686322176}, {system_total_memory,4294500352}]}, {statistics, [{wall_clock,{1132672,0}}, {context_switches,{158050,0}}, {garbage_collection,{24210,115452087,0}}, {io,{{input,10589364},{output,8875088}}}, {reductions,{30421724,250264}}, {run_queue,0}, {runtime,{41609,656}}]}]}] [ns_server:error] [2012-05-14 13:56:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default": {'EXIT',{noproc,{gen_server,call, [{'stats_reader-default','ns_1@10.1.3.83'}, {latest,minute}]}}}