=PROGRESS REPORT==== 14-May-2012::13:56:18 ===
supervisor: {local,sasl_safe_sup}
started: [{pid,<0.31.0>},
{name,alarm_handler},
{mfargs,{alarm_handler,start_link,[]}},
{restart_type,permanent},
{shutdown,2000},
{child_type,worker}]
=PROGRESS REPORT==== 14-May-2012::13:56:18 ===
supervisor: {local,sasl_safe_sup}
started: [{pid,<0.32.0>},
{name,overload},
{mfargs,{overload,start_link,[]}},
{restart_type,permanent},
{shutdown,2000},
{child_type,worker}]
=PROGRESS REPORT==== 14-May-2012::13:56:18 ===
supervisor: {local,sasl_sup}
started: [{pid,<0.30.0>},
{name,sasl_safe_sup},
{mfargs,
{supervisor,start_link,
[{local,sasl_safe_sup},sasl,safe]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
=PROGRESS REPORT==== 14-May-2012::13:56:18 ===
supervisor: {local,sasl_sup}
started: [{pid,<0.33.0>},
{name,release_handler},
{mfargs,{release_handler,start_link,[]}},
{restart_type,permanent},
{shutdown,2000},
{child_type,worker}]
=PROGRESS REPORT==== 14-May-2012::13:56:18 ===
application: sasl
started_at: nonode@nohost
[ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:<0.67.0>:ns_server:init_logging:138] Started & configured logging
[ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:<0.67.0>:ns_server:log_pending:30] Static config terms:
[{error_logger_mf_dir,"C:\\Program Files\\Couchbase\\Server\\var\\lib\\couchbase\\logs"},
{error_logger_mf_maxbytes,10485760},
{error_logger_mf_maxfiles,20},
{path_config_bindir,"C:\\Program Files\\Couchbase\\Server\\bin"},
{path_config_etcdir,"C:\\Program Files\\Couchbase\\Server\\etc\\couchbase"},
{path_config_libdir,"C:\\Program Files\\Couchbase\\Server\\lib"},
{path_config_datadir,"C:\\Program Files\\Couchbase\\Server\\var\\lib\\couchbase"},
{path_config_tmpdir,"C:\\Program Files\\Couchbase\\Server\\var\\lib\\couchbase\\tmp"},
{loglevel_default,info},
{loglevel_ns_server,info},
{loglevel_error_logger,info},
{loglevel_user,info},
{loglevel_menelaus,info},
{loglevel_ns_doctor,info},
{loglevel_stats,info},
{loglevel_rebalance,info},
{loglevel_cluster,info}]
[ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:ns_server_cluster_sup:log_os_info:start_link:25] OS type: {win32,nt} Version: {6,1,7600}
Runtime info: [{otp_release,"R14B02"},
{erl_version,"5.8.3"},
{erl_version_long,
"Erlang R14B02 (erts-5.8.3) [smp:4:4] [rq:4] [async-threads:16]\n"},
{system_arch_raw,"win32"},
{system_arch,"windows"},
{localtime,{{2012,5,14},{13,37,16}}},
{memory,
[{total,6256136},
{processes,898408},
{processes_used,890112},
{system,5357728},
{atom,446233},
{atom_used,438204},
{binary,15896},
{code,3414830},
{ets,193220}]},
{loaded,
[ns_info,log_os_info,cb_init_loggers,ns_server_cluster_sup,
ale_default_formatter,otp_internal,misc,
'ale_logger-cluster','ale_logger-rebalance',
'ale_logger-stats','ale_logger-ns_doctor',
'ale_logger-menelaus','ale_logger-user',
'ale_logger-ns_server',ns_log_sink,disk_log_sup,
disk_log_server,disk_log_1,disk_log,ale_disk_sink,
ns_server,timer,io_lib_fread,memsup,disksup,
os_mon_sysinfo,os_mon,sasl_report,release_handler,
calendar,overload,alarm_handler,sasl_report_tty_h,sasl,
ale_error_logger_handler,'ale_logger-ale_logger',
'ale_logger-error_logger',beam_opcodes,beam_dict,beam_asm,
beam_validator,beam_flatten,beam_trim,beam_receive,
beam_bsm,beam_peep,beam_dead,beam_type,beam_bool,
beam_clean,beam_utils,beam_jump,beam_block,v3_codegen,
v3_life,v3_kernel,sys_core_dsetel,erl_bifs,sys_core_fold,
cerl_trees,sys_core_inline,core_lib,cerl,v3_core,erl_bits,
erl_expand_records,sys_pre_expand,sofs,erl_internal,
compile,dynamic_compile,ale_utils,io_lib_pretty,
ale_codegen,ale,ale_dynamic_sup,ale_sup,ale_app,
io_lib_format,io_lib,io,sets,dict,ordsets,erl_lint,
ram_file,beam_lib,ns_bootstrap,file_io_server,orddict,
erl_eval,file,c,error_logger_tty_h,kernel_config,queue,
shell,user,user_drv,user_sup,supervisor_bridge,
standard_error,unicode,binary,ets,gb_sets,
hipe_unified_loader,packages,code_server,code,file_server,
global_group,gen_tcp,inet_tcp,auth,erl_epmd,inet_tcp_dist,
net_kernel,erl_distribution,filename,win32reg,inet_parse,
inet,inet_udp,os,inet_config,inet_db,global,gb_trees,rpc,
supervisor,kernel,application_master,sys,application,
gen_server,erl_parse,proplists,erl_scan,lists,
application_controller,proc_lib,gen,gen_event,
error_logger,heart,error_handler,erlang,erl_prim_loader,
prim_zip,zlib,prim_file,prim_inet,init,otp_ring0]},
{applications,
[{ale,"Another Logger for Erlang","8cffe61"},
{os_mon,"CPO CXC 138 46","2.2.5"},
{kernel,"ERTS CXC 138 10","2.14.3"},
{sasl,"SASL CXC 138 11","2.1.9.3"},
{ns_server,"Couchbase server","1.8.1-815-rel-enterprise"},
{stdlib,"ERTS CXC 138 10","1.17.3"}]},
{pre_loaded,
[erlang,erl_prim_loader,prim_zip,zlib,prim_file,prim_inet,
init,otp_ring0]},
{process_count,63},
{node,'ns_1@10.1.3.83'},
{nodes,[]},
{registered,
[init,'sink-disk_debug','sink-disk_error',inet_db,rex,
net_sup,kernel_sup,global_name_server,ale_sup,
kernel_safe_sup,release_handler,file_server_2,
ale_dynamic_sup,overload,global_group,alarm_handler,
'sink-disk_default',erl_prim_loader,erl_epmd,
sasl_safe_sup,auth,error_logger,net_kernel,disk_log_sup,
disk_log_server,os_mon_sup,ns_server_cluster_sup,
code_server,os_mon_sysinfo,application_controller,memsup,
disksup,standard_error,ale,timer_server,
standard_error_sup,sasl_sup,user,'sink-ns_log']},
{cookie,nocookie},
{wordsize,4},
{wall_clock,1}]
[ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:ns_server_cluster_sup:log_os_info:start_link:27] Manifest:
["","",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" ",
" "," ",
" ",
" ",
" ",""]
[error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.128.0>},
{name,timeout_diag_logger},
{mfargs,{timeout_diag_logger,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:read_address_config:52] reading ip config from "c:/Program Files/Couchbase/Server/var/lib/couchbase/ip"
[ns_server:error] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:read_address_config:58] Got error:einval. Ignoring bad address:[]
[ns_server:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:bringup:145] Attempting to bring up net_kernel with name 'ns_1@127.0.0.1'
[ns_server:debug] [2012-05-14 13:37:16] [ns_1@10.1.3.83:dist_manager:dist_manager:bringup:153] Attempted to save node name to disk: undefined
[error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.129.0>},
{name,dist_manager},
{mfargs,{dist_manager,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.130.0>},
{name,ns_cookie_manager},
{mfargs,{ns_cookie_manager,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.131.0>},
{name,ns_cluster},
{mfargs,{ns_cluster,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mb_mnesia_sup}
started: [{pid,<0.133.0>},
{name,mb_mnesia_events},
{mfargs,
{gen_event,start_link,[{local,mb_mnesia_events}]}},
{restart_type,permanent},
{shutdown,10},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_sup}
started: [{pid,<0.140.0>},
{name,mnesia_event},
{mfargs,{mnesia_sup,start_event,[]}},
{restart_type,permanent},
{shutdown,30000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.142.0>},
{name,mnesia_monitor},
{mfargs,{mnesia_monitor,start,[]}},
{restart_type,permanent},
{shutdown,3000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.143.0>},
{name,mnesia_subscr},
{mfargs,{mnesia_subscr,start,[]}},
{restart_type,permanent},
{shutdown,3000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.144.0>},
{name,mnesia_locker},
{mfargs,{mnesia_locker,start,[]}},
{restart_type,permanent},
{shutdown,3000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.145.0>},
{name,mnesia_recover},
{mfargs,{mnesia_recover,start,[]}},
{restart_type,permanent},
{shutdown,180000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.146.0>},
{name,mnesia_tm},
{mfargs,{mnesia_tm,start,[]}},
{restart_type,permanent},
{shutdown,30000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.147.0>},
{name,mnesia_checkpoint_sup},
{mfargs,{mnesia_checkpoint_sup,start,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.148.0>},
{name,mnesia_snmp_sup},
{mfargs,{mnesia_snmp_sup,start,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.149.0>},
{name,mnesia_controller},
{mfargs,{mnesia_controller,start,[]}},
{restart_type,permanent},
{shutdown,3000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_kernel_sup}
started: [{pid,<0.150.0>},
{name,mnesia_late_loader},
{mfargs,{mnesia_late_loader,start,[]}},
{restart_type,permanent},
{shutdown,3000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mnesia_sup}
started: [{pid,<0.141.0>},
{name,mnesia_kernel_sup},
{mfargs,{mnesia_kernel_sup,start,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
application: mnesia
started_at: 'ns_1@10.1.3.83'
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,kernel_safe_sup}
started: [{pid,<0.156.0>},
{name,dets_sup},
{mfargs,{dets_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,kernel_safe_sup}
started: [{pid,<0.157.0>},
{name,dets},
{mfargs,{dets_server,start_link,[]}},
{restart_type,permanent},
{shutdown,2000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:ensure_schema:410] Committed schema to disk.
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:init:246] Current config:
[{access_module,mnesia},
{auto_repair,true},
{backup_module,mnesia_backup},
{checkpoints,[]},
{db_nodes,['ns_1@10.1.3.83']},
{debug,verbose},
{directory,"c:/Program Files/Couchbase/Server/var/lib/couchbase/mnesia"},
{dump_log_load_regulation,false},
{dump_log_time_threshold,180000},
{dump_log_update_in_place,true},
{dump_log_write_threshold,1000},
{embedded_mnemosyne,false},
{event_module,mnesia_event},
{extra_db_nodes,[]},
{fallback_activated,false},
{held_locks,[]},
{ignore_fallback_at_startup,false},
{fallback_error_function,{mnesia,lkill}},
{is_running,yes},
{local_tables,[local_config,cluster,schema]},
{lock_queue,[]},
{log_version,"4.3"},
{master_node_tables,[]},
{max_wait_for_decision,10000},
{protocol_version,{7,6}},
{running_db_nodes,['ns_1@10.1.3.83']},
{schema_location,opt_disc},
{schema_version,{3,0}},
{subscribers,[<0.140.0>,<0.134.0>]},
{tables,[local_config,cluster,schema]},
{transaction_commits,5},
{transaction_failures,0},
{transaction_log_writes,3},
{transaction_restarts,0},
{transactions,[]},
{use_dir,true},
{core_dir,false},
{no_table_loaders,2},
{dc_dump_limit,4},
{send_compressed,0},
{version,"4.4.17"}]
Peers: ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:167] Info from Mnesia:
Create Directory "c:/Program Files/Couchbase/Server/var/lib/couchbase/mnesia"
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mb_mnesia_sup}
started: [{pid,<0.134.0>},
{name,mb_mnesia},
{mfargs,{mb_mnesia,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,schema,
[{name,schema},
{type,set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,false},
{record_name,schema},
{attributes,[table,cstruct]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,114000},'ns_1@10.1.3.83'}},
{version,{{3,0},{'ns_1@10.1.3.83',{1337,27837,177002}}}}]},
{tid,3,<0.153.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:195] Peers: ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:195] Peers: ['ns_1@10.1.3.83']
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.132.0>},
{name,mb_mnesia_sup},
{mfargs,{mb_mnesia_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,local_config,
[{name,local_config},
{type,set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,local_config},
{attributes,[key,val]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,333001},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,5,<0.176.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,local_config,
[{name,local_config},
{type,set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,local_config},
{attributes,[key,val]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,333001},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,5,<0.176.0>}}
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_sup:ns_config_sup:init:32] loading static ns_config from "c:/Program Files/Couchbase/Server/etc/couchbase/config"
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_config_sup}
started: [{pid,<0.183.0>},
{name,ns_config_events},
{mfargs,
{gen_event,start_link,[{local,ns_config_events}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:load_config:614] Loading static config from "c:/Program Files/Couchbase/Server/etc/couchbase/config"
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:load_config:628] Loading dynamic config from "c:/Program Files/Couchbase/Server/var/lib/couchbase/config/config.dat"
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:load_config:632] No dynamic config file found. Assuming we're brand new node
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_6_to_1_7:278] Upgrading config from 1.6 to 1.7
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes:
[{set,{node,'ns_1@10.1.3.83',config_version},{1,7}},
{set,
{node,'ns_1@10.1.3.83',memcached},
[{port,11210},
{dedicated_port,11209},
{dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"},
{admin_user,"_admin"},
{admin_pass,"_admin"},
{bucket_engine,
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"},
{engines,
[{membase,
[{engine,
"c:/Program Files/Couchbase/Server/lib/memcached/ep.so"},
{initfile,
"c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"},
{static_config_string,
"vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]},
{memcached,
[{engine,
"c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"},
{static_config_string,"vb0=true"}]}]},
{verbosity,[]}]},
{set,
{node,'ns_1@10.1.3.83',ns_log},
[{filename,
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}]},
{set,
{node,'ns_1@10.1.3.83',isasl},
[{path,
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}]},
{set,directory,"c:/Program Files/Couchbase/Server/var/lib/couchbase/config"}]
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_7_to_1_7_1:301] Upgrading config from 1.7 to 1.7.1
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes:
[{set,{node,'ns_1@10.1.3.83',config_version},{1,7,1}},
{set,email_alerts,
[{recipients,["root@localhost"]},
{sender,"couchbase@localhost"},
{enabled,false},
{email_server,[{user,[]},
{pass,[]},
{host,"localhost"},
{port,25},
{encrypt,false}]},
{alerts,[auto_failover_node,auto_failover_maximum_reached,
auto_failover_other_nodes_down,
auto_failover_cluster_too_small]}]},
{set,auto_failover_cfg,
[{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}]
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_7_1_to_1_7_2:312] Upgrading config from 1.7.1 to 1.7.2
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes:
[{set,{node,'ns_1@10.1.3.83',config_version},{1,7,2}}]
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_7_2_to_1_8_0:366] Upgrading config from 1.7.2 to 1.8.0
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes:
[{set,{node,'ns_1@10.1.3.83',config_version},{1,8,0}},
{set,{node,'ns_1@10.1.3.83',port_servers},
[{moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
{"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
[port]},
"-z",
{"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming",
[{misc,this_node_rest_port,[]}]},
"-p","0","-Y","y","-O","stderr",
{"~s",[verbosity]}],
[{env,[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR",
{"~s",[{ns_moxi_sup,rest_user,[]}]}},
{"MOXI_SASL_PLAIN_PWD",
{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]},
use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,
stream]},
{memcached,"c:/Program Files/Couchbase/Server/bin/memcached",
["-X",
"c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so",
"-l",
{"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]},
"-E",
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so",
"-B","binary","-r","-c","10000","-e",
{"admin=~s;default_bucket_name=default;auto_create=false",
[admin_user]},
{"~s",[verbosity]}],
[{env,[{"EVENT_NOSELECT","1"},
{"MEMCACHED_TOP_KEYS","100"},
{"ISASL_PWFILE",{"~s",[{isasl,path}]}},
{"ISASL_DB_CHECK_TIME","1"}]},
use_stdio,stderr_to_stdout,exit_status,
port_server_send_eol,stream]}]}]
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config_default:upgrade_config_from_1_8_0_to_1_8_1:403] Upgrading config from 1.8.0 to 1.8.1
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_upgrade_config:420] Upgrading config by changes:
[{set,{node,'ns_1@10.1.3.83',config_version},{1,8,1}},
{set,vbucket_map_history,[]}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config:ns_config:do_init:465] Upgraded initial config:
{config,
{full,"c:/Program Files/Couchbase/Server/etc/couchbase/config",undefined,
ns_config_default},
[[],
[{directory,"c:/Program Files/Couchbase/Server/var/lib/couchbase/config"},
{nodes_wanted,['ns_1@10.1.3.83']},
{{node,'ns_1@10.1.3.83',membership},active},
{rest,[{port,8091}]},
{{node,'ns_1@10.1.3.83',rest},[{port,8091},{port_meta,global}]},
{rest_creds,[{creds,[]}]},
{{node,'ns_1@10.1.3.83',isasl},
[{path,
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}]},
{{node,'ns_1@10.1.3.83',memcached},
[{port,11210},
{dedicated_port,11209},
{dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"},
{admin_user,"_admin"},
{admin_pass,"_admin"},
{bucket_engine,
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"},
{engines,
[{membase,
[{engine,
"c:/Program Files/Couchbase/Server/lib/memcached/ep.so"},
{initfile,
"c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"},
{static_config_string,
"vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]},
{memcached,
[{engine,
"c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"},
{static_config_string,"vb0=true"}]}]},
{verbosity,[]}]},
{memory_quota,3276},
{buckets,[{configs,[]}]},
{{node,'ns_1@10.1.3.83',moxi},[{port,11211},{verbosity,[]}]},
{{node,'ns_1@10.1.3.83',port_servers},
[{moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
{"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
[port]},
"-z",
{"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming",
[{misc,this_node_rest_port,[]}]},
"-p","0","-Y","y","-O","stderr",
{"~s",[verbosity]}],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}},
{"MOXI_SASL_PLAIN_PWD",
{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]},
use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,
stream]},
{memcached,"c:/Program Files/Couchbase/Server/bin/memcached",
["-X",
"c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so",
"-l",
{"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]},
"-E",
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so",
"-B","binary","-r","-c","10000","-e",
{"admin=~s;default_bucket_name=default;auto_create=false",
[admin_user]},
{"~s",[verbosity]}],
[{env,
[{"EVENT_NOSELECT","1"},
{"MEMCACHED_TOP_KEYS","100"},
{"ISASL_PWFILE",{"~s",[{isasl,path}]}},
{"ISASL_DB_CHECK_TIME","1"}]},
use_stdio,stderr_to_stdout,exit_status,port_server_send_eol,
stream]}]},
{{node,'ns_1@10.1.3.83',ns_log},
[{filename,
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}]},
{email_alerts,
[{recipients,["root@localhost"]},
{sender,"couchbase@localhost"},
{enabled,false},
{email_server,
[{user,[]},
{pass,[]},
{host,"localhost"},
{port,25},
{encrypt,false}]},
{alerts,
[auto_failover_node,auto_failover_maximum_reached,
auto_failover_other_nodes_down,
auto_failover_cluster_too_small]}]},
{replication,[{enabled,true}]},
{auto_failover_cfg,
[{enabled,false},{timeout,30},{max_nodes,1},{count,0}]}]],
[[{vbucket_map_history,[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]}]},
{directory,
[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]},
99,58,47,80,114,111,103,114,97,109,32,70,105,108,101,115,47,67,111,
117,99,104,98,97,115,101,47,83,101,114,118,101,114,47,118,97,114,
47,108,105,98,47,99,111,117,99,104,98,97,115,101,47,99,111,110,102,
105,103]},
{{node,'ns_1@10.1.3.83',config_version},
[{'_vclock',[{'ns_1@10.1.3.83',{5,63504247037}}]}|{1,8,1}]},
{auto_failover_cfg,
[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]},
{enabled,false},
{timeout,30},
{max_nodes,1},
{count,0}]},
{buckets,[{configs,[]}]},
{email_alerts,
[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]},
{recipients,["root@localhost"]},
{sender,"couchbase@localhost"},
{enabled,false},
{email_server,
[{user,[]},
{pass,[]},
{host,"localhost"},
{port,25},
{encrypt,false}]},
{alerts,
[auto_failover_node,auto_failover_maximum_reached,
auto_failover_other_nodes_down,
auto_failover_cluster_too_small]}]},
{memory_quota,3276},
{nodes_wanted,['ns_1@10.1.3.83']},
{replication,[{enabled,true}]},
{rest,[{port,8091}]},
{rest_creds,[{creds,[]}]},
{{node,'ns_1@10.1.3.83',isasl},
[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]},
{path,
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}]},
{{node,'ns_1@10.1.3.83',membership},active},
{{node,'ns_1@10.1.3.83',memcached},
[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]},
{port,11210},
{dedicated_port,11209},
{dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"},
{admin_user,"_admin"},
{admin_pass,"_admin"},
{bucket_engine,
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"},
{engines,
[{membase,
[{engine,
"c:/Program Files/Couchbase/Server/lib/memcached/ep.so"},
{initfile,
"c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"},
{static_config_string,
"vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]},
{memcached,
[{engine,
"c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"},
{static_config_string,"vb0=true"}]}]},
{verbosity,[]}]},
{{node,'ns_1@10.1.3.83',moxi},[{port,11211},{verbosity,[]}]},
{{node,'ns_1@10.1.3.83',ns_log},
[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]},
{filename,
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}]},
{{node,'ns_1@10.1.3.83',port_servers},
[{'_vclock',[{'ns_1@10.1.3.83',{1,63504247037}}]},
{moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
{"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
[port]},
"-z",
{"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming",
[{misc,this_node_rest_port,[]}]},
"-p","0","-Y","y","-O","stderr",
{"~s",[verbosity]}],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}},
{"MOXI_SASL_PLAIN_PWD",
{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]},
use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,
stream]},
{memcached,"c:/Program Files/Couchbase/Server/bin/memcached",
["-X",
"c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so",
"-l",
{"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]},
"-E",
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so",
"-B","binary","-r","-c","10000","-e",
{"admin=~s;default_bucket_name=default;auto_create=false",
[admin_user]},
{"~s",[verbosity]}],
[{env,
[{"EVENT_NOSELECT","1"},
{"MEMCACHED_TOP_KEYS","100"},
{"ISASL_PWFILE",{"~s",[{isasl,path}]}},
{"ISASL_DB_CHECK_TIME","1"}]},
use_stdio,stderr_to_stdout,exit_status,port_server_send_eol,
stream]}]},
{{node,'ns_1@10.1.3.83',rest},[{port,8091},{port_meta,global}]}]],
ns_config_default,
{ns_config,save_config_sync,[]},
undefined,false}
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_config_sup}
started: [{pid,<0.184.0>},
{name,ns_config},
{mfargs,
{ns_config,start_link,
["c:/Program Files/Couchbase/Server/etc/couchbase/config",
ns_config_default]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_config_sup}
started: [{pid,<0.187.0>},
{name,ns_config_remote},
{mfargs,
{ns_config_replica,start_link,
[{local,ns_config_remote}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:init:49] isasl_sync init: ["c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw",
"_admin","_admin"]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:init:57] isasl_sync init buckets: []
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_config_sup}
started: [{pid,<0.188.0>},
{name,ns_config_isasl_sync},
{mfargs,{ns_config_isasl_sync,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_config_sup}
started: [{pid,<0.191.0>},
{name,ns_config_log},
{mfargs,{ns_config_log,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.182.0>},
{name,ns_config_sup},
{mfargs,{ns_config_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.193.0>},
{name,vbucket_filter_changes_registry},
{mfargs,
{ns_process_registry,start_link,
[vbucket_filter_changes_registry]}},
{restart_type,permanent},
{shutdown,100},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_log:ns_log:init:64] Couldn't load logs from "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log" (perhaps it's first startup): {error,
enoent}
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.195.0>},
{name,ns_log},
{mfargs,{ns_log,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.196.0>},
{name,ns_log_events},
{mfargs,{gen_event,start_link,[{local,ns_log_events}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_node_disco_sup}
started: [{pid,<0.198.0>},
{name,ns_node_disco_events},
{mfargs,
{gen_event,start_link,
[{local,ns_node_disco_events}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_node_disco:ns_node_disco:init:98] Initting ns_node_disco with []
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_sync:115] ns_cookie_manager do_cookie_sync
[user:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_init:91] Initial otp cookie generated: caonwcesmjjqmzcs
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
otp ->
[{cookie,caonwcesmjjqmzcs}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.200.0>:ns_node_disco:do_nodes_wanted_updated_fun:182] ns_node_disco: nodes_wanted updated: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.200.0>:ns_node_disco:do_nodes_wanted_updated_fun:188] ns_node_disco: nodes_wanted pong: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_node_disco_sup}
started: [{pid,<0.199.0>},
{name,ns_node_disco},
{mfargs,{ns_node_disco,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_node_disco_sup}
started: [{pid,<0.203.0>},
{name,ns_node_disco_log},
{mfargs,{ns_node_disco_log,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_node_disco_sup}
started: [{pid,<0.204.0>},
{name,ns_node_disco_conf_events},
{mfargs,{ns_node_disco_conf_events,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:init:58] init pulling
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:init:60] init pushing
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:init:64] init reannouncing
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:50] ns_node_disco_conf_events config on otp
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
otp ->
[{cookie,caonwcesmjjqmzcs}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_sync:115] ns_cookie_manager do_cookie_sync
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:44] ns_node_disco_conf_events config on nodes_wanted
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
vbucket_map_history ->
[]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
directory ->
"c:/Program Files/Couchbase/Server/var/lib/couchbase/config"
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.207.0>:ns_node_disco:do_nodes_wanted_updated_fun:182] ns_node_disco: nodes_wanted updated: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_cookie_manager:ns_cookie_manager:do_cookie_sync:115] ns_cookie_manager do_cookie_sync
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.207.0>:ns_node_disco:do_nodes_wanted_updated_fun:188] ns_node_disco: nodes_wanted pong: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',config_version} ->
{1,8,1}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
auto_failover_cfg ->
[{enabled,false},{timeout,30},{max_nodes,1},{count,0}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.208.0>:ns_node_disco:do_nodes_wanted_updated_fun:182] ns_node_disco: nodes_wanted updated: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[]}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.208.0>:ns_node_disco:do_nodes_wanted_updated_fun:188] ns_node_disco: nodes_wanted pong: ['ns_1@10.1.3.83'], with cookie: caonwcesmjjqmzcs
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
email_alerts ->
[{recipients,["root@localhost"]},
{sender,"couchbase@localhost"},
{enabled,false},
{email_server,[{user,[]},
{pass,[]},
{host,"localhost"},
{port,25},
{encrypt,false}]},
{alerts,[auto_failover_node,auto_failover_maximum_reached,
auto_failover_other_nodes_down,auto_failover_cluster_too_small]}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
memory_quota ->
3276
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
nodes_wanted ->
['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
replication ->
[{enabled,true}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
rest ->
[{port,8091}]
[ns_server:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:handle_info:57] config change: rest_creds -> ********
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',isasl} ->
[{path,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',membership} ->
active
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',memcached} ->
[{port,11210},
{dedicated_port,11209},
{dbdir,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data"},
{admin_user,"_admin"},
{admin_pass,"_admin"},
{bucket_engine,
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so"},
{engines,
[{membase,
[{engine,"c:/Program Files/Couchbase/Server/lib/memcached/ep.so"},
{initfile,
"c:/Program Files/Couchbase/Server/etc/couchbase/init.sql"},
{static_config_string,
"vb0=false;waitforwarmup=false;failpartialwarmup=false;shardpattern=%d/%b-%i.mb;db_strategy=multiMTVBDB"}]},
{memcached,
[{engine,
"c:/Program Files/Couchbase/Server/lib/memcached/default_engine.so"},
{static_config_string,"vb0=true"}]}]},
{verbosity,[]}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',moxi} ->
[{port,11211},{verbosity,[]}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',ns_log} ->
[{filename,"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/ns_log"}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',port_servers} ->
[{moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
{"port_listen=~B,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
[port]},
"-z",
{"url=http://127.0.0.1:~B/pools/default/saslBucketsStreaming",
[{misc,this_node_rest_port,[]}]},
"-p","0","-Y","y","-O","stderr",
{"~s",[verbosity]}],
[{env,[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR",{"~s",[{ns_moxi_sup,rest_user,[]}]}},
{"MOXI_SASL_PLAIN_PWD",{"~s",[{ns_moxi_sup,rest_pass,[]}]}}]},
use_stdio,exit_status,port_server_send_eol,stderr_to_stdout,stream]},
{memcached,"c:/Program Files/Couchbase/Server/bin/memcached",
["-X",
"c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so",
"-l",
{"0.0.0.0:~B,0.0.0.0:~B:1000",[port,dedicated_port]},
"-E",
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so",
"-B","binary","-r","-c","10000","-e",
{"admin=~s;default_bucket_name=default;auto_create=false",
[admin_user]},
{"~s",[verbosity]}],
[{env,[{"EVENT_NOSELECT","1"},
{"MEMCACHED_TOP_KEYS","100"},
{"ISASL_PWFILE",{"~s",[{isasl,path}]}},
{"ISASL_DB_CHECK_TIME","1"}]},
use_stdio,stderr_to_stdout,exit_status,port_server_send_eol,
stream]}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
{node,'ns_1@10.1.3.83',rest} ->
[{port,8091},{port_meta,global}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_node_disco_sup}
started: [{pid,<0.205.0>},
{name,ns_config_rep},
{mfargs,{ns_config_rep,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.197.0>},
{name,ns_node_disco_sup},
{mfargs,{ns_node_disco_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.210.0>},
{name,ns_tick_event},
{mfargs,{gen_event,start_link,[{local,ns_tick_event}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.211.0>},
{name,mb_master_events},
{mfargs,
{gen_event,start_link,[{local,mb_master_events}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_server_sup:mb_master:check_master_takeover_needed:150] Was unable to discover master, not going to force mastership takeover
[user:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master:mb_master:init:89] I'm the only node, so I'm the master.
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master_sup:misc:start_singleton:824] start_singleton(gen_fsm, ns_orchestrator, [], []): started as <0.218.0> on 'ns_1@10.1.3.83'
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mb_master_sup}
started: [{pid,<0.218.0>},
{name,ns_orchestrator},
{mfargs,{ns_orchestrator,start_link,[]}},
{restart_type,permanent},
{shutdown,20},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master_sup:misc:start_singleton:824] start_singleton(gen_server, ns_tick, [], []): started as <0.219.0> on 'ns_1@10.1.3.83'
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mb_master_sup}
started: [{pid,<0.219.0>},
{name,ns_tick},
{mfargs,{ns_tick,start_link,[]}},
{restart_type,permanent},
{shutdown,10},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.220.0>:auto_failover:init:120] init auto_failover.
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_master_sup:misc:start_singleton:824] start_singleton(gen_server, auto_failover, [], []): started as <0.220.0> on 'ns_1@10.1.3.83'
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,mb_master_sup}
started: [{pid,<0.220.0>},
{name,auto_failover},
{mfargs,{auto_failover,start_link,[]}},
{restart_type,permanent},
{shutdown,10},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.215.0>},
{name,mb_master},
{mfargs,{mb_master,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.221.0>},
{name,master_activity_events},
{mfargs,
{gen_event,start_link,
[{local,master_activity_events}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.222.0>},
{name,master_activity_events_ingress},
{mfargs,
{gen_event,start_link,
[{local,master_activity_events_ingress}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.223.0>},
{name,master_activity_events_timestamper},
{mfargs,
{master_activity_events,start_link_timestamper,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.224.0>},
{name,master_activity_events_keeper},
{mfargs,{master_activity_events_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.226.0>},
{name,master_activity_events_srv},
{mfargs,
{master_activity_events,start_link_work_server,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.227.0>},
{name,buckets_events},
{mfargs,
{gen_event,start_link,[{local,buckets_events}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_mail_sup}
started: [{pid,<0.229.0>},
{name,ns_mail},
{mfargs,{ns_mail,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_log_events:ns_mail_log:init:45] ns_mail_log started up
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_mail_sup}
started: [{pid,<0.230.0>},
{name,ns_mail_log},
{mfargs,{ns_mail_log,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.228.0>},
{name,ns_mail_sup},
{mfargs,{ns_mail_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.231.0>},
{name,ns_heart},
{mfargs,{ns_heart,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.235.0>},
{name,ns_doctor},
{mfargs,{ns_doctor,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:144] Failed to grab system stats:
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-@system','ns_1@10.1.3.83'},
{latest,"minute"}]}}}
[ns_doctor:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:89] Got initial status [{'ns_1@10.1.3.83',
[{last_heard,{1337,27837,708000}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,9768936},
{processes,2306208},
{processes_used,2300160},
{system,7462728},
{atom,641577},
{atom_used,636189},
{binary,78184},
{code,5148278},
{ets,297676}]},
{system_stats,
[{cpu_utilization_rate,0},
{swap_total,0},
{swap_used,0}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,2},
{memory_data,{4294500352,697974784,{<0.7.0>,159012}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3631923200},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{2656,812}},
{context_switches,{20357,0}},
{garbage_collection,{5231,18586032,0}},
{io,{{input,5464203},{output,772365}}},
{reductions,{10646300,10646300}},
{run_queue,0},
{runtime,{2593,2593}}]}]}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.238.0>},
{name,menelaus_web},
{mfargs,{menelaus_web,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.240.0>},
{name,menelaus_event},
{mfargs,{menelaus_event,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.241.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[user:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:ns_server_sup:menelaus_sup:start_link:44] Couchbase Server has started on web port 8091 on node 'ns_1@10.1.3.83'.
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.242.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.237.0>},
{name,menelaus},
{mfargs,{menelaus_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_port_sup}
started: [{pid,<0.244.0>},
{name,ns_port_init},
{mfargs,{ns_port_init,start_link,[]}},
{restart_type,permanent},
{shutdown,10},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.245.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:<0.247.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_port_sup}
started: [{pid,<0.245.0>},
{name,
{moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR",[]},
{"MOXI_SASL_PLAIN_PWD",[]}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}},
{mfargs,
{supervisor_cushion,start_link,
[moxi,5000,ns_port_server,start_link,
[moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR",[]},
{"MOXI_SASL_PLAIN_PWD",[]}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]]]}},
{restart_type,permanent},
{shutdown,10000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_port_sup}
started: [{pid,<0.247.0>},
{name,
{memcached,
"c:/Program Files/Couchbase/Server/bin/memcached",
["-X",
"c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so",
"-l","0.0.0.0:11210,0.0.0.0:11209:1000","-E",
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so",
"-B","binary","-r","-c","10000","-e",
"admin=_admin;default_bucket_name=default;auto_create=false",
[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MEMCACHED_TOP_KEYS","100"},
{"ISASL_PWFILE",
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"},
{"ISASL_DB_CHECK_TIME","1"}]},
use_stdio,stderr_to_stdout,exit_status,
port_server_send_eol,stream]}},
{mfargs,
{supervisor_cushion,start_link,
[memcached,5000,ns_port_server,start_link,
[memcached,
"c:/Program Files/Couchbase/Server/bin/memcached",
["-X",
"c:/Program Files/Couchbase/Server/lib/memcached/stdin_term_handler.so",
"-l","0.0.0.0:11210,0.0.0.0:11209:1000",
"-E",
"c:/Program Files/Couchbase/Server/lib/memcached/bucket_engine.so",
"-B","binary","-r","-c","10000","-e",
"admin=_admin;default_bucket_name=default;auto_create=false",
[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MEMCACHED_TOP_KEYS","100"},
{"ISASL_PWFILE",
"c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"},
{"ISASL_DB_CHECK_TIME","1"}]},
use_stdio,stderr_to_stdout,exit_status,
port_server_send_eol,stream]]]}},
{restart_type,permanent},
{shutdown,10000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.243.0>},
{name,ns_port_sup},
{mfargs,{ns_port_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,60000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.249.0>},
{name,ns_stats_event},
{mfargs,
{gen_event,start_link,[{local,ns_stats_event}]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.250.0>},
{name,ns_bucket_worker},
{mfargs,{work_queue,start_link,[ns_bucket_worker]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.252.0>},
{name,buckets_observing_subscription},
{mfargs,{ns_bucket_sup,subscribe_on_config_events,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.251.0>},
{name,ns_bucket_sup},
{mfargs,{ns_bucket_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.253.0>},
{name,system_stats_collector},
{mfargs,{system_stats_collector,start_link,[]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.255.0>},
{name,{stats_archiver,"@system"}},
{mfargs,{stats_archiver,start_link,["@system"]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-minute'
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.257.0>},
{name,{stats_reader,"@system"}},
{mfargs,{stats_reader,start_link,["@system"]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.258.0>},
{name,ns_moxi_sup_work_queue},
{mfargs,
{work_queue,start_link,[ns_moxi_sup_work_queue]}},
{restart_type,permanent},
{shutdown,1000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_sup}
started: [{pid,<0.259.0>},
{name,ns_moxi_sup},
{mfargs,{ns_moxi_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_server_cluster_sup}
started: [{pid,<0.194.0>},
{name,ns_server_sup},
{mfargs,{ns_server_sup,start_link,[]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:37:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
application: ns_server
started_at: 'ns_1@10.1.3.83'
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-minute',
[{name,'stats_archiver-@system-minute'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,848027},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,6,<0.261.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-minute',
[{name,'stats_archiver-@system-minute'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,848027},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,6,<0.261.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-hour'
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-hour',
[{name,'stats_archiver-@system-hour'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,848040},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,7,<0.269.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-hour',
[{name,'stats_archiver-@system-hour'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,848040},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,7,<0.269.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-day'
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-day',
[{name,'stats_archiver-@system-day'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,864005},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,8,<0.276.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-day',
[{name,'stats_archiver-@system-day'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,864005},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,8,<0.276.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-week'
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-week',
[{name,'stats_archiver-@system-week'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,864014},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,9,<0.283.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-week',
[{name,'stats_archiver-@system-week'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,864014},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,9,<0.283.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-month'
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-month',
[{name,'stats_archiver-@system-month'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,880007},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,10,<0.290.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-month',
[{name,'stats_archiver-@system-month'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,880007},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,10,<0.290.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_call:109] Creating table 'stats_archiver-@system-year'
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-year',
[{name,'stats_archiver-@system-year'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,895005},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,11,<0.297.0>}}
[ns_server:debug] [2012-05-14 13:37:17] [ns_1@10.1.3.83:mb_mnesia:mb_mnesia:handle_info:200] Mnesia table event:
{write,{schema,'stats_archiver-@system-year',
[{name,'stats_archiver-@system-year'},
{type,ordered_set},
{ram_copies,[]},
{disc_copies,['ns_1@10.1.3.83']},
{disc_only_copies,[]},
{load_order,0},
{access_mode,read_write},
{index,[]},
{snmp,[]},
{local_content,true},
{record_name,stat_entry},
{attributes,[timestamp,values]},
{user_properties,[]},
{frag_properties,[]},
{cookie,{{1337,27837,895005},'ns_1@10.1.3.83'}},
{version,{{2,0},[]}}]},
{tid,11,<0.297.0>}}
[ns_server:debug] [2012-05-14 13:37:18] [ns_1@10.1.3.83:<0.239.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61452 path /pools/default/saslBucketsStreaming
[error_logger:info] [2012-05-14 13:37:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,inet_gethost_native_sup}
started: [{pid,<0.308.0>},{mfa,{inet_gethost_native,init,[[]]}}]
[error_logger:info] [2012-05-14 13:37:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,kernel_safe_sup}
started: [{pid,<0.307.0>},
{name,inet_gethost_native_sup},
{mfargs,{inet_gethost_native,start_link,[]}},
{restart_type,temporary},
{shutdown,1000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_port_sup:terminate_port:87] unsupervising port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR",[]},
{"MOXI_SASL_PLAIN_PWD",[]}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}
[ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:<0.246.0>:ns_port_server:log:161] moxi<0.246.0>: EOL on stdin. Exiting
[menelaus:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:<0.239.0>:menelaus_web:handle_streaming:707] menelaus_web streaming socket closed by client
[ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_port_sup:launch_port:54] supervising port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:<0.393.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000
[error_logger:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_port_sup}
started: [{pid,<0.393.0>},
{name,
{moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}},
{mfargs,
{supervisor_cushion,start_link,
[moxi,5000,ns_port_server,start_link,
[moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]]]}},
{restart_type,permanent},
{shutdown,10000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
rest ->
[{port,8091}]
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:info] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_log:ns_config_log:handle_info:57] config change: rest_creds -> ********
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
memory_quota ->
3276
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:37:52] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:debug] [2012-05-14 13:37:53] [ns_1@10.1.3.83:<0.397.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61516 path /pools/default/saslBucketsStreaming
[ns_server:info] [2012-05-14 13:37:53] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: 2012-05-14 13:37:53: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13)
moxi<0.394.0>: 2012-05-14 13:37:53: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8)
[ns_doctor:debug] [2012-05-14 13:38:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,27892,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,11723872},
{processes,3704776},
{processes_used,3691432},
{system,8019096},
{atom,696501},
{atom_used,669919},
{binary,154664},
{code,5484890},
{ets,381220}]},
{system_stats,
[{cpu_utilization_rate,25.0},
{swap_total,9850605568},
{swap_used,748527616}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,57},
{memory_data,{4294500352,697974784,{<0.7.0>,159012}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3642757120},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{47656,0}},
{context_switches,{28995,0}},
{garbage_collection,{6928,22353488,0}},
{io,{{input,7300017},{output,2437947}}},
{reductions,{11996122,220631}},
{run_queue,0},
{runtime,{3843,281}}]}]}]
[ns_doctor:debug] [2012-05-14 13:39:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,27952,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,11642400},
{processes,3610708},
{processes_used,3597364},
{system,8031692},
{atom,696501},
{atom_used,669919},
{binary,149800},
{code,5484890},
{ets,398700}]},
{system_stats,
[{cpu_utilization_rate,25.03173394262503},
{swap_total,9850605568},
{swap_used,745025536}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,117},
{memory_data,{4294500352,650600448,{<0.231.0>,300620}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3648258048},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{107656,0}},
{context_switches,{33168,0}},
{garbage_collection,{7540,23817698,0}},
{io,{{input,7309230},{output,2518291}}},
{reductions,{12481563,129508}},
{run_queue,0},
{runtime,{4890,250}}]}]}]
[ns_doctor:debug] [2012-05-14 13:40:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28012,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,11910000},
{processes,3851912},
{processes_used,3838568},
{system,8058088},
{atom,696501},
{atom_used,669919},
{binary,160312},
{code,5484890},
{ets,414524}]},
{system_stats,
[{cpu_utilization_rate,24.993649987299975},
{swap_total,9850605568},
{swap_used,745504768}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,177},
{memory_data,{4294500352,645046272,{<0.231.0>,300620}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3650203648},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{167656,0}},
{context_switches,{37410,0}},
{garbage_collection,{8036,25360744,0}},
{io,{{input,7318452},{output,2602984}}},
{reductions,{12982172,129654}},
{run_queue,0},
{runtime,{6031,328}}]}]}]
[ns_doctor:debug] [2012-05-14 13:41:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28072,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,11986384},
{processes,3918620},
{processes_used,3904796},
{system,8067764},
{atom,696501},
{atom_used,669919},
{binary,153184},
{code,5484890},
{ets,431460}]},
{system_stats,
[{cpu_utilization_rate,31.378522467631377},
{swap_total,9850605568},
{swap_used,754688000}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,237},
{memory_data,{4294500352,642867200,{<0.397.0>,300760}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3655122944},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{227656,0}},
{context_switches,{41980,0}},
{garbage_collection,{8641,26977742,0}},
{io,{{input,7571009},{output,2902350}}},
{reductions,{13522125,128746}},
{run_queue,0},
{runtime,{7343,312}}]}]}]
[ns_doctor:debug] [2012-05-14 13:42:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28132,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,12834560},
{processes,4494112},
{processes_used,4483536},
{system,8340448},
{atom,700405},
{atom_used,677771},
{binary,330912},
{code,5554827},
{ets,448932}]},
{system_stats,
[{cpu_utilization_rate,25.374650749301498},
{swap_total,9850605568},
{swap_used,771563520}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,297},
{memory_data,{4294500352,644669440,{<0.231.0>,300620}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3637673984},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{287656,0}},
{context_switches,{48329,0}},
{garbage_collection,{9609,30315266,0}},
{io,{{input,8937383},{output,4372830}}},
{reductions,{14585061,691292}},
{run_queue,0},
{runtime,{8609,469}}]}]}]
[ns_doctor:debug] [2012-05-14 13:43:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28192,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,12342456},
{processes,4022036},
{processes_used,4009476},
{system,8320420},
{atom,700405},
{atom_used,677771},
{binary,297656},
{code,5554827},
{ets,464220}]},
{system_stats,
[{cpu_utilization_rate,25.38109756097561},
{swap_total,9850605568},
{swap_used,766648320}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,357},
{memory_data,{4294500352,656515072,{<0.397.0>,300760}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3640819712},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{347656,0}},
{context_switches,{53069,0}},
{garbage_collection,{10197,33026302,0}},
{io,{{input,8946864},{output,4463522}}},
{reductions,{15167674,130303}},
{run_queue,0},
{runtime,{10000,407}}]}]}]
[menelaus:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.1274.0>:menelaus_web_buckets:do_bucket_create:262] Created bucket "default" of type: membase
[{num_replicas,1},{ram_quota,1048576000},{auth_type,sasl}]
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[[{map,[]},
{fastForwardMap,[]},
{sasl_password,[]},
{num_replicas,1},
{ram_quota,1048576000},
{auth_type,sasl},
{type,membase},
{num_vbuckets,1024},
{servers,[]}]]}]
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:84] Starting new child: {{per_bucket_sup,"default"},
{single_bucket_sup,start_link,["default"]},
permanent,infinity,supervisor,
[single_bucket_sup]}
[ns_server:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[{"default",
[{map,[]},
{fastForwardMap,[]},
{sasl_password,[]},
{num_replicas,1},
{ram_quota,1048576000},
{auth_type,sasl},
{type,membase},
{num_vbuckets,1024},
{servers,['ns_1@10.1.3.83']}]}]}]
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:44:04] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[error_logger:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1315.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[ns_server:info] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: 2012-05-14 13:44:07: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Empty serverList ({
moxi<0.394.0>: "name": "default",
moxi<0.394.0>: "nodeLocator": "vbucket",
moxi<0.394.0>: "saslPassword": "",
moxi<0.394.0>: "nodes": [],
moxi<0.394.0>: "vBucketServerMap": {
moxi<0.394.0>: "hashAlgorithm": "CRC",
moxi<0.394.0>: "numReplicas": 1,
moxi<0.394.0>: "serverList": [],
moxi<0.394.0>: "vBucketMap": []
moxi<0.394.0>: }
moxi<0.394.0>: })
moxi<0.394.0>: 2012-05-14 13:44:07: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({
moxi<0.394.0>: "name": "default",
moxi<0.394.0>: "nodeLocator": "vbucket",
moxi<0.394.0>: "saslPassword": "",
moxi<0.394.0>: "nodes": [{
moxi<0.394.0>: "replication": "undefined",
moxi<0.394.0>: "clusterMembership": "active",
moxi<0.394.0>: "status": "warmup",
moxi<0.394.0>: "hostname": "10.1.3.83:8091",
moxi<0.394.0>: "clusterCompatibility": 1,
moxi<0.394.0>: "version": "1.8.1-815-rel-enterprise",
moxi<0.394.0>: "os": "windows",
moxi<0.394.0>: "ports": {
moxi<0.394.0>: "proxy": 11211,
moxi<0.394.0>: "direct": 11210
moxi<0.394.0>: }
moxi<0.394.0>: }],
moxi<0.394.0>: "vBucketServerMap": {
moxi<0.394.0>: "hashAlgorithm": "CRC",
moxi<0.394.0>: "numReplicas": 1,
moxi<0.394.0>: "serverList": ["10.1.3.83:11210"],
moxi<0.394.0>: "vBucketMap": []
moxi<0.394.0>: }
moxi<0.394.0>: })
[stats:error] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:44:04] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[menelaus:warn] [2012-05-14 13:44:05] [ns_1@10.1.3.83:<0.972.0>:menelaus_web:loop:273] Client-side error-report for user "Administrator" on node 'ns_1@10.1.3.83':
User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19
Got unhandled error: Uncaught Error: BUG
At: http://10.1.3.83:8091/js/cells.js:64
Backtrace:
Function: collectBacktraceViaCaller
Args:
---------
Function: appOnError
Args:
"Uncaught Error: BUG"
"http://10.1.3.83:8091/js/cells.js"
64
---------
[ns_server:warn] [2012-05-14 13:44:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.242.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1329.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:44:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1307.0>}
[ns_server:warn] [2012-05-14 13:44:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:44:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:44:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1329.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1334.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:09] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:44:10] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1334.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1346.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:44:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:info] [2012-05-14 13:44:12] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: 2012-05-14 13:44:15: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({
moxi<0.394.0>: "name": "default",
moxi<0.394.0>: "nodeLocator": "vbucket",
moxi<0.394.0>: "saslPassword": "",
moxi<0.394.0>: "nodes": [{
moxi<0.394.0>: "replication": 1,
moxi<0.394.0>: "clusterMembership": "active",
moxi<0.394.0>: "status": "warmup",
moxi<0.394.0>: "hostname": "10.1.3.83:8091",
moxi<0.394.0>: "clusterCompatibility": 1,
moxi<0.394.0>: "version": "1.8.1-815-rel-enterprise",
moxi<0.394.0>: "os": "windows",
moxi<0.394.0>: "ports": {
moxi<0.394.0>: "proxy": 11211,
moxi<0.394.0>: "direct": 11210
moxi<0.394.0>: }
moxi<0.394.0>: }],
moxi<0.394.0>: "vBucketServerMap": {
moxi<0.394.0>: "hashAlgorithm": "CRC",
moxi<0.394.0>: "numReplicas": 1,
moxi<0.394.0>: "serverList": ["10.1.3.83:11210"],
moxi<0.394.0>: "vBucketMap": []
moxi<0.394.0>: }
moxi<0.394.0>: })
[stats:error] [2012-05-14 13:44:12] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1317.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1315.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1325.0>,
{#Ref<0.0.0.13178>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.13278>},connected},
{'$gen_call',{<0.1332.0>,#Ref<0.0.0.13280>},topkeys},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1344.0>,
{#Ref<0.0.0.13480>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.13554>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.1316.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7355
neighbours:
[error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1315.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1315.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1355.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.241.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1356.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1346.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1362.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:15] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:44:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1307.0>}
[ns_server:error] [2012-05-14 13:44:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_doctor:debug] [2012-05-14 13:44:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28254,723008}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13236424},
{processes,4925688},
{processes_used,4914864},
{system,8310736},
{atom,701869},
{atom_used,679949},
{binary,238368},
{code,5581197},
{ets,485348}]},
{system_stats,
[{cpu_utilization_rate,26.618345413646587},
{swap_total,9850605568},
{swap_used,765308928}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,419},
{memory_data,{4294500352,653561856,{<0.231.0>,300620}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3641167872},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{407656,0}},
{context_switches,{58372,0}},
{garbage_collection,{10903,36345610,0}},
{io,{{input,9202417},{output,4639563}}},
{reductions,{15864473,207793}},
{run_queue,0},
{runtime,{11421,328}}]}]}]
[error_logger:error] [2012-05-14 13:44:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1362.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1375.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:18] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:44:20] [ns_1@10.1.3.83:<0.1307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:44:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1375.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1384.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:21] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:44:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:44:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1384.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1399.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:<0.1274.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1360.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1355.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1365.0>,
{#Ref<0.0.0.13807>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.13955>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.1382.0>,#Ref<0.0.0.14103>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1387.0>,
{#Ref<0.0.0.14160>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.14290>},connected},
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.1357.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1355.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1403.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1356.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1405.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1355.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[stats:error] [2012-05-14 13:44:24] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1399.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1416.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:44:27] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1307.0>}
[ns_server:error] [2012-05-14 13:44:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:44:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:28] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:44:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1416.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1426.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:31] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:44:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:44:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1426.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1437.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:34] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:35] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1406.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1403.0>]
messages: [{'$gen_call',{<0.1307.0>,#Ref<0.0.0.14410>},
list_vbuckets_prevstate},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.14625>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.1427.0>,#Ref<0.0.0.14771>},topkeys},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.14927>},connected},
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.1404.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7350
neighbours:
[error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.1307.0>
registered_name: []
exception exit: {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4298
neighbours:
[error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1403.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1403.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1447.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1405.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1450.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1437.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:35] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1455.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:37] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:44:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:44:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1455.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1466.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:40] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:44:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1466.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1477.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:44:42] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:44:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:44:43] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:44] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1477.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:44] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1488.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:44:45] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1449.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1447.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.1464.0>,
{#Ref<0.0.0.15285>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.15309>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.1469.0>,#Ref<0.0.0.15372>},topkeys},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.15557>},connected},
check_started,check_started,
{'$gen_call',{<0.1486.0>,
{#Ref<0.0.0.15658>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started]
links: [<0.64.0>,<0.1448.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7349
neighbours:
[error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1447.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1447.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1489.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1450.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1492.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:46] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:44:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1462.0>}
[ns_server:error] [2012-05-14 13:44:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:44:47] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1488.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:47] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1505.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:49] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1505.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1514.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:44:51] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:44:52] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:44:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:44:53] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1514.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:53] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1529.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:44:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:44:55] [ns_1@10.1.3.83:<0.1462.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1491.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1489.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1497.0>,
{#Ref<0.0.0.15771>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.15910>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.1512.0>,#Ref<0.0.0.16052>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1517.0>,
{#Ref<0.0.0.16109>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.16235>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.1490.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1492.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1489.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1535.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1534.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:44:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1489.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[ns_server:warn] [2012-05-14 13:44:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:56] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1529.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:56] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1544.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:44:57] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1462.0>}
[ns_server:error] [2012-05-14 13:44:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:44:58] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:44:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:44:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1544.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:44:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1554.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:45:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:45:01] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:45:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:45:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:45:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1554.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:45:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1565.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.1280.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:45:04] [ns_1@10.1.3.83:<0.972.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:45:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:45:05] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1537.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1535.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1542.0>,
{#Ref<0.0.0.16457>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.16524>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.1555.0>,#Ref<0.0.0.16670>},topkeys},
check_started,check_started,
{'$gen_call',{<0.1462.0>,#Ref<0.0.0.16691>},
list_vbuckets_prevstate},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.16827>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.1536.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7354
neighbours:
[error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.1462.0>
registered_name: []
exception exit: {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4289
neighbours:
[error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1535.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1534.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1535.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1585.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1584.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1565.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:45:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1598.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:45:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:45:07] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:45:07] [ns_1@10.1.3.83:<0.1568.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:45:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:45:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:45:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1598.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:45:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1611.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:45:09] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason shutdown
[ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:91] Stopping child for dead bucket: {{per_bucket_sup,"default"},
<0.1585.0>,supervisor,
[single_bucket_sup]}
[ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[]}]
[ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"
[ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:45:09] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:warn] [2012-05-14 13:45:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:45:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:45:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1585.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: shutdown_error
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1585.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:45:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1587.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1585.0>]
messages: [check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1609.0>,
{#Ref<0.0.0.17358>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.17380>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.17697>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.1586.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7344
neighbours:
[ns_doctor:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28315,645003}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,13309552},
{processes,4937016},
{processes_used,4925752},
{system,8372536},
{atom,702357},
{atom_used,681053},
{binary,267416},
{code,5596508},
{ets,501260}]},
{system_stats,
[{cpu_utilization_rate,27.368157960509873},
{swap_total,9850605568},
{swap_used,764198912}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,480},
{memory_data,{4294500352,653295616,{<0.972.0>,300780}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3643469824},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{470484,0}},
{context_switches,{68919,0}},
{garbage_collection,{12196,46153944,0}},
{io,{{input,9261751},{output,5005366}}},
{reductions,{17345273,293759}},
{run_queue,0},
{runtime,{14453,594}}]}]}]
[ns_server:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:idle:373] Restarting moxi on nodes ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1618.0>:ns_pubsub:do_subscribe_link:132] Parent process exited with reason normal
[ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1618.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.17517>} event handler: ok
[ns_server:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1645.0>:ns_port_sup:restart_port:92] restarting port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}
[ns_server:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.394.0>:ns_port_server:log:161] moxi<0.394.0>: EOL on stdin. Exiting
[menelaus:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.397.0>:menelaus_web:handle_streaming:707] menelaus_web streaming socket closed by client
[ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1647.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000
[menelaus:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1568.0>:menelaus_web_buckets:handle_bucket_delete:192] Deleted bucket "default"
[error_logger:info] [2012-05-14 13:45:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_port_sup}
started: [{pid,<0.1647.0>},
{name,
{moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}},
{mfargs,
{supervisor_cushion,start_link,
[moxi,5000,ns_port_server,start_link,
[moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]]]}},
{restart_type,permanent},
{shutdown,10000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:45:17] [ns_1@10.1.3.83:<0.1620.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61725 path /pools/default/saslBucketsStreaming
[ns_server:info] [2012-05-14 13:45:18] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: 2012-05-14 13:45:17: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13)
moxi<0.1648.0>: 2012-05-14 13:45:17: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8)
[menelaus:info] [2012-05-14 13:45:57] [ns_1@10.1.3.83:<0.1736.0>:menelaus_web_buckets:do_bucket_create:262] Created bucket "default" of type: membase
[{num_replicas,1},{ram_quota,1048576000},{auth_type,sasl}]
[ns_server:info] [2012-05-14 13:45:57] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[[{map,[]},
{fastForwardMap,[]},
{sasl_password,[]},
{num_replicas,1},
{ram_quota,1048576000},
{auth_type,sasl},
{type,membase},
{num_vbuckets,1024},
{servers,[]}]]}]
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:84] Starting new child: {{per_bucket_sup,"default"},
{single_bucket_sup,start_link,["default"]},
permanent,infinity,supervisor,
[single_bucket_sup]}
[error_logger:info] [2012-05-14 13:45:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1768.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[{"default",
[{map,[]},
{fastForwardMap,[]},
{sasl_password,[]},
{num_replicas,1},
{ram_quota,1048576000},
{auth_type,sasl},
{type,membase},
{num_vbuckets,1024},
{servers,['ns_1@10.1.3.83']}]}]}]
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[stats:error] [2012-05-14 13:45:57] [ns_1@10.1.3.83:<0.1736.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:45:57] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:info] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: 2012-05-14 13:45:59: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({
moxi<0.1648.0>: "name": "default",
moxi<0.1648.0>: "nodeLocator": "vbucket",
moxi<0.1648.0>: "saslPassword": "",
moxi<0.1648.0>: "nodes": [{
moxi<0.1648.0>: "replication": "undefined",
moxi<0.1648.0>: "clusterMembership": "active",
moxi<0.1648.0>: "status": "warmup",
moxi<0.1648.0>: "hostname": "10.1.3.83:8091",
moxi<0.1648.0>: "clusterCompatibility": 1,
moxi<0.1648.0>: "version": "1.8.1-815-rel-enterprise",
moxi<0.1648.0>: "os": "windows",
moxi<0.1648.0>: "ports": {
moxi<0.1648.0>: "proxy": 11211,
moxi<0.1648.0>: "direct": 11210
moxi<0.1648.0>: }
moxi<0.1648.0>: }],
moxi<0.1648.0>: "vBucketServerMap": {
moxi<0.1648.0>: "hashAlgorithm": "CRC",
moxi<0.1648.0>: "numReplicas": 1,
moxi<0.1648.0>: "serverList": ["10.1.3.83:11210"],
moxi<0.1648.0>: "vBucketMap": []
moxi<0.1648.0>: }
moxi<0.1648.0>: })
[stats:error] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1671.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:45:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:45:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:45:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1611.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:45:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1791.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:00] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:46:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:46:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1791.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1801.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:46:03] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:46:03] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1801.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:05] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1812.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:06] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:46:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1765.0>}
[ns_server:error] [2012-05-14 13:46:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:info] [2012-05-14 13:46:07] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: 2012-05-14 13:46:09: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({
moxi<0.1648.0>: "name": "default",
moxi<0.1648.0>: "nodeLocator": "vbucket",
moxi<0.1648.0>: "saslPassword": "",
moxi<0.1648.0>: "nodes": [{
moxi<0.1648.0>: "replication": 1,
moxi<0.1648.0>: "clusterMembership": "active",
moxi<0.1648.0>: "status": "warmup",
moxi<0.1648.0>: "hostname": "10.1.3.83:8091",
moxi<0.1648.0>: "clusterCompatibility": 1,
moxi<0.1648.0>: "version": "1.8.1-815-rel-enterprise",
moxi<0.1648.0>: "os": "windows",
moxi<0.1648.0>: "ports": {
moxi<0.1648.0>: "proxy": 11211,
moxi<0.1648.0>: "direct": 11210
moxi<0.1648.0>: }
moxi<0.1648.0>: }],
moxi<0.1648.0>: "vBucketServerMap": {
moxi<0.1648.0>: "hashAlgorithm": "CRC",
moxi<0.1648.0>: "numReplicas": 1,
moxi<0.1648.0>: "serverList": ["10.1.3.83:11210"],
moxi<0.1648.0>: "vBucketMap": []
moxi<0.1648.0>: }
moxi<0.1648.0>: })
[error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1770.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1768.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1787.0>,
{#Ref<0.0.0.20455>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.1797.0>,#Ref<0.0.0.20622>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.20680>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.1808.0>,
{#Ref<0.0.0.20820>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.21031>},connected},
check_started]
links: [<0.64.0>,<0.1769.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7355
neighbours:
[error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1768.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1768.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1823.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1584.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1824.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1812.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1830.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:46:09] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1830.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1841.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:12] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:46:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:46:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:46:14] [ns_1@10.1.3.83:<0.1765.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:46:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1841.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:14] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1850.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:46:15] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:46:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1765.0>}
[ns_doctor:debug] [2012-05-14 13:46:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28368,20008}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13761960},
{processes,5346264},
{processes_used,5336072},
{system,8415696},
{atom,702357},
{atom_used,681123},
{binary,298024},
{code,5596508},
{ets,514300}]},
{system_stats,
[{cpu_utilization_rate,27.418126428027417},
{swap_total,9850605568},
{swap_used,770727936}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,533},
{memory_data,{4294500352,650952704,{<0.1280.0>,300780}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3640164352},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{532672,0}},
{context_switches,{78096,0}},
{garbage_collection,{13420,56277027,0}},
{io,{{input,9277932},{output,5261376}}},
{reductions,{18692868,548379}},
{run_queue,0},
{runtime,{16953,953}}]}]}]
[ns_server:error] [2012-05-14 13:46:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:46:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1850.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:17] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1880.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:<0.1736.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:18] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1828.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1823.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1831.0>,
{#Ref<0.0.0.21154>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.21428>},connected},
check_started,
{'$gen_call',{<0.1846.0>,#Ref<0.0.0.21431>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1851.0>,
{#Ref<0.0.0.21491>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.21759>},connected},
check_started]
links: [<0.64.0>,<0.1827.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.1765.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4288
neighbours:
[error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1824.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1884.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1823.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1823.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:46:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1885.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[stats:error] [2012-05-14 13:46:18] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1880.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1899.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:46:21] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:46:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:46:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1899.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:23] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1910.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:24] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1910.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:26] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1918.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:46:27] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:46:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:info] [2012-05-14 13:46:28] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1887.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1885.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.22179>},connected},
check_started,check_started,
{'$gen_call',{<0.1906.0>,#Ref<0.0.0.22185>},topkeys},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1927.0>,
{#Ref<0.0.0.22481>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.22511>},connected},
check_started,check_started]
links: [<0.64.0>,<0.1886.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7347
neighbours:
[error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1885.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1930.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1885.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1884.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1933.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1918.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1940.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:30] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:46:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:46:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1940.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1948.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:46:33] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:46:34] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1948.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1960.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:36] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:46:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1925.0>}
[ns_server:error] [2012-05-14 13:46:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:46:38] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1930.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.1971.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1932.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1930.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1936.0>,
{#Ref<0.0.0.22589>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.22778>},connected},
check_started,check_started,
{'$gen_call',{<0.1949.0>,#Ref<0.0.0.22803>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1956.0>,
{#Ref<0.0.0.22927>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.23118>},connected},
check_started,check_started]
links: [<0.64.0>,<0.1931.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1930.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1933.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1974.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1960.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1978.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:39] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:46:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1978.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1989.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:42] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:46:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:46:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:46:44] [ns_1@10.1.3.83:<0.1925.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:46:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1989.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.1998.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:45] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:46:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.1925.0>}
[ns_server:warn] [2012-05-14 13:46:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:46:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.1998.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2011.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:48] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.1973.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.1971.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.1982.0>,
{#Ref<0.0.0.23263>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.23478>},connected},
check_started,check_started,
{'$gen_call',{<0.1994.0>,#Ref<0.0.0.23489>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.1999.0>,
{#Ref<0.0.0.23545>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.23718>},connected},
check_started,check_started]
links: [<0.64.0>,<0.1972.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.1971.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2016.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.1974.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.1971.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.1925.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4286
neighbours:
[error_logger:info] [2012-05-14 13:46:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2017.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2011.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2027.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:51] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:46:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:46:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2027.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2038.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:46:54] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:46:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2038.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:46:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2046.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:46:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:46:57] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:46:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:46:58] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2019.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2016.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.24078>},connected},
check_started,check_started,
{'$gen_call',{<0.2036.0>,#Ref<0.0.0.24100>},topkeys},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.2051.0>,
{#Ref<0.0.0.24299>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.24326>},connected},
check_started,check_started]
links: [<0.64.0>,<0.2018.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7347
neighbours:
[error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2016.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2058.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2017.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2016.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:46:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2059.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:46:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2046.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2068.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:00] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:47:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:47:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2068.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2076.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:03] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:04] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2076.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2088.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:06] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:47:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2049.0>}
[ns_server:error] [2012-05-14 13:47:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:47:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:08] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2061.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2058.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2066.0>,
{#Ref<0.0.0.24508>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.24679>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.2083.0>,#Ref<0.0.0.24799>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2086.0>,
{#Ref<0.0.0.24854>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.25018>},connected},
check_started,check_started,check_started,
{'EXIT',<0.2060.0>,shutdown}]
links: [<0.64.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7355
neighbours:
[error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2058.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2099.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2058.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2059.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:08] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2101.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2088.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2106.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:09] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:47:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2106.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2117.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:47:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:47:12] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:15] [ns_1@10.1.3.83:<0.2049.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:47:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2117.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2126.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:15] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:16] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2049.0>}
[ns_doctor:debug] [2012-05-14 13:47:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28428,973010}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13375976},
{processes,4945240},
{processes_used,4933480},
{system,8430736},
{atom,702357},
{atom_used,681123},
{binary,296360},
{code,5596508},
{ets,531324}]},
{system_stats,
[{cpu_utilization_rate,42.60369815092454},
{swap_total,9850605568},
{swap_used,728403968}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,593},
{memory_data,{4294500352,654356480,{<0.1620.0>,415388}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3683676160},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{592672,0}},
{context_switches,{87884,0}},
{garbage_collection,{14613,64538730,0}},
{io,{{input,9535379},{output,5805069}}},
{reductions,{20013748,362705}},
{run_queue,0},
{runtime,{19718,843}}]}]}]
[ns_server:error] [2012-05-14 13:47:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:47:18] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2126.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2141.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:18] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:19] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2102.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2099.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2113.0>,
{#Ref<0.0.0.25207>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.25296>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.2124.0>,#Ref<0.0.0.25404>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2133.0>,
{#Ref<0.0.0.25546>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.25622>},connected},
check_started,check_started,check_started]
links: [<0.64.0>,<0.2100.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.2049.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4288
neighbours:
[error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2099.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2099.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2146.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2101.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2148.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:20] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2141.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2157.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:21] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:22] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:47:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:47:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2157.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2168.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:24] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:47:24] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:26] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2168.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2176.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:47:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:47:27] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:28] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:29] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2149.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2146.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.26005>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.2166.0>,#Ref<0.0.0.26036>},topkeys},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.2181.0>,
{#Ref<0.0.0.26232>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.26255>},connected},
check_started,check_started,check_started]
links: [<0.64.0>,<0.2147.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7347
neighbours:
[error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2146.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2188.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2148.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2190.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2146.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:47:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2176.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2197.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:30] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:47:31] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:47:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:47:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2197.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2206.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:34] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:35] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:47:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2206.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2217.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:47:37] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:47:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2179.0>}
[ns_server:error] [2012-05-14 13:47:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:47:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2217.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2230.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2191.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2188.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2198.0>,
{#Ref<0.0.0.26453>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.26606>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2213.0>,#Ref<0.0.0.26726>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2218.0>,
{#Ref<0.0.0.26791>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.26937>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.2189.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2190.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2231.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2188.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2188.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:47:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2232.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[stats:error] [2012-05-14 13:47:40] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2230.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2247.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:47:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:47:43] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2247.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2256.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:47:45] [ns_1@10.1.3.83:<0.2179.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:47:46] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2179.0>}
[ns_server:error] [2012-05-14 13:47:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:47:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2256.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2269.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:49] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2234.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2232.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2243.0>,
{#Ref<0.0.0.27132>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.27220>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2254.0>,#Ref<0.0.0.27331>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2263.0>,
{#Ref<0.0.0.27469>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.27533>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.2233.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2232.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.2179.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4286
neighbours:
[error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2231.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2232.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2277.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:47:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2276.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2269.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2285.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:47:52] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:47:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:47:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2285.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2295.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:47:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:47:55] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:47:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2295.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2304.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:47:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:47:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:47:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:47:59] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2279.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2277.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.27899>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.2296.0>,#Ref<0.0.0.27963>},topkeys},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.2309.0>,
{#Ref<0.0.0.28128>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.28155>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.2278.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7349
neighbours:
[error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2277.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2318.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2276.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2321.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:47:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2277.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:48:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2304.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2325.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:01] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:48:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:48:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2325.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2334.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:48:04] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:48:05] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2334.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2345.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:07] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:48:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2307.0>}
[ns_server:error] [2012-05-14 13:48:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:48:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2345.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2358.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:48:10] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2320.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2318.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2326.0>,
{#Ref<0.0.0.28363>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.28499>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.2341.0>,#Ref<0.0.0.28637>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2346.0>,
{#Ref<0.0.0.28705>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.28841>},connected},
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.2319.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7357
neighbours:
[error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2321.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2318.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2318.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2362.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:48:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2361.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:48:10] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2358.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2375.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:48:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:48:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:13] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2375.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2384.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:48:16] [ns_1@10.1.3.83:<0.2307.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:48:16] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:48:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2307.0>}
[ns_doctor:debug] [2012-05-14 13:48:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28490,20010}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13265808},
{processes,4880500},
{processes_used,4868220},
{system,8385308},
{atom,702357},
{atom_used,681123},
{binary,264088},
{code,5596508},
{ets,517852}]},
{system_stats,
[{cpu_utilization_rate,27.768057985503624},
{swap_total,9850605568},
{swap_used,719425536}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,655},
{memory_data,{4294500352,602001408,{<0.1620.0>,415388}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3693133824},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{652672,0}},
{context_switches,{97160,0}},
{garbage_collection,{15745,72376666,0}},
{io,{{input,9545962},{output,6119630}}},
{reductions,{21272892,369478}},
{run_queue,0},
{runtime,{22687,844}}]}]}]
[ns_server:error] [2012-05-14 13:48:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:48:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2384.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2399.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:19] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:20] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2364.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2362.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2371.0>,
{#Ref<0.0.0.29045>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.29116>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.2382.0>,#Ref<0.0.0.29247>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2392.0>,
{#Ref<0.0.0.29395>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.29445>},connected},
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.2363.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2362.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2406.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.2307.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4287
neighbours:
[error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2361.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2362.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:48:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2408.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2399.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2415.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:48:22] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:48:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:48:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2415.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2425.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:25] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2425.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2434.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:48:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:48:28] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2409.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2406.0>]
messages: [check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.29812>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.2426.0>,#Ref<0.0.0.29877>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2439.0>,
{#Ref<0.0.0.30038>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.30069>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2407.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7349
neighbours:
[error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2406.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2406.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2448.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2408.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2451.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2434.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2455.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:31] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:48:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:48:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2455.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2464.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:48:34] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:48:36] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2464.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2475.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:48:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2437.0>}
[ns_server:error] [2012-05-14 13:48:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:48:37] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2475.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2488.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:48:40] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2450.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2448.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2456.0>,
{#Ref<0.0.0.30288>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.30417>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.2471.0>,#Ref<0.0.0.30556>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2476.0>,
{#Ref<0.0.0.30619>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.30671>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2449.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2448.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2448.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2491.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2451.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2493.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:48:40] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2488.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2505.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:48:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:48:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:43] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2505.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2514.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:48:46] [ns_1@10.1.3.83:<0.2437.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:48:46] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:48:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2437.0>}
[ns_server:error] [2012-05-14 13:48:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:48:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2514.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2527.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:49] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:50] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2494.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2491.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2501.0>,
{#Ref<0.0.0.30967>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.31038>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.2512.0>,#Ref<0.0.0.31167>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2521.0>,
{#Ref<0.0.0.31309>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.31354>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2492.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2491.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2534.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.2437.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4286
neighbours:
[error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2493.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2536.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:48:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2491.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:48:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2527.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2543.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:48:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:48:52] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:48:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2543.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2553.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:48:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:55] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:48:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2553.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:48:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2565.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:48:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:48:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:48:58] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:48:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2565.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2577.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2537.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2534.0>]
messages: [check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.31636>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.2556.0>,#Ref<0.0.0.31803>},topkeys},
check_started,check_started,check_started,
{'$gen_call',{<0.2567.0>,
{#Ref<0.0.0.31933>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.31967>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2535.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7349
neighbours:
[error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2534.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2534.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2578.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2536.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2580.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:01] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:49:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:49:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2577.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2594.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:05] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2594.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2603.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:49:06] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:49:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2564.0>}
[ns_server:error] [2012-05-14 13:49:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:49:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:08] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2603.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2618.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:49:10] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2578.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2619.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2578.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2583.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2578.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2586.0>,
{#Ref<0.0.0.32198>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.32314>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.2601.0>,#Ref<0.0.0.32479>},topkeys},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.32573>},connected},
check_started,
{'$gen_call',{<0.2610.0>,
{#Ref<0.0.0.32574>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2579.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7358
neighbours:
[error_logger:error] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2580.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2622.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:49:11] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2618.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2633.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:49:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:49:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:14] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2633.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2643.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:49:16] [ns_1@10.1.3.83:<0.2564.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:49:17] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:49:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2564.0>}
[ns_doctor:debug] [2012-05-14 13:49:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28550,927009}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13424632},
{processes,4987580},
{processes_used,4975268},
{system,8437052},
{atom,702357},
{atom_used,681123},
{binary,299792},
{code,5596508},
{ets,533852}]},
{system_stats,
[{cpu_utilization_rate,26.979765176117912},
{swap_total,9850605568},
{swap_used,720330752}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,715},
{memory_data,{4294500352,601063424,{<0.1620.0>,415388}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3694972928},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{712672,0}},
{context_switches,{106253,0}},
{garbage_collection,{16890,79899708,0}},
{io,{{input,9556449},{output,6430954}}},
{reductions,{22509285,340449}},
{run_queue,0},
{runtime,{25500,688}}]}]}]
[ns_server:error] [2012-05-14 13:49:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:49:18] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2643.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2670.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:20] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:20] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:21] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2621.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2619.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2631.0>,
{#Ref<0.0.0.32897>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.32937>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.2644.0>,#Ref<0.0.0.33096>},topkeys},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.33347>},connected},
check_started,
{'$gen_call',{<0.2668.0>,
{#Ref<0.0.0.33372>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2620.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7353
neighbours:
[error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.2564.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4288
neighbours:
[error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2622.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2677.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2619.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2678.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2619.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2670.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2688.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:22] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:49:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:49:23] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:24] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2688.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2696.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:26] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:26] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2696.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2710.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:49:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:49:28] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:29] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:30] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2710.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2720.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:49:31] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2680.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2678.0>]
messages: [check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.33653>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.2699.0>,#Ref<0.0.0.33824>},topkeys},
check_started,check_started,check_started,
{'$gen_call',{<0.2708.0>,
{#Ref<0.0.0.33951>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.33987>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started]
links: [<0.64.0>,<0.2679.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7350
neighbours:
[error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2678.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2721.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2677.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2723.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2678.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[ns_server:warn] [2012-05-14 13:49:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:32] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:49:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:49:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2720.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2737.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:35] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2737.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2746.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:49:37] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:49:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2706.0>}
[ns_server:error] [2012-05-14 13:49:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:49:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:38] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2746.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2761.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:49:41] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2724.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2721.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2729.0>,
{#Ref<0.0.0.34229>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.34344>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2744.0>,#Ref<0.0.0.34506>},topkeys},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.34589>},connected},
check_started,check_started,
{'$gen_call',{<0.2753.0>,
{#Ref<0.0.0.34595>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2722.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7353
neighbours:
[error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2721.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2721.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2765.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2723.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:41] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2767.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:49:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:49:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2761.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2776.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:49:44] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2776.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2786.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:49:47] [ns_1@10.1.3.83:<0.2706.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:49:47] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:49:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2706.0>}
[ns_server:error] [2012-05-14 13:49:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:49:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2786.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2798.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:49:50] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:49:51] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2768.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2765.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.2774.0>,
{#Ref<0.0.0.34915>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.34945>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2787.0>,#Ref<0.0.0.35121>},topkeys},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.35265>},connected},
check_started,check_started,
{'$gen_call',{<0.2796.0>,
{#Ref<0.0.0.35275>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2766.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7353
neighbours:
[error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2765.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.2706.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4286
neighbours:
[error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2765.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2805.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2767.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2806.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2798.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2816.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:<0.1744.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:49:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:49:53] [ns_1@10.1.3.83:<0.1734.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:49:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:49:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2816.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:54] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2836.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:49:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:49:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2836.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:49:57] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2848.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:49:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:50:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2848.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2852.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:50:01] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2805.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2855.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2805.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2808.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2805.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.35734>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2839.0>,#Ref<0.0.0.35883>},topkeys},
check_started,check_started,
{'$gen_call',{<0.2844.0>,
{#Ref<0.0.0.35917>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.35948>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.2807.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2806.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2858.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:50:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:50:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2852.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:03] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2865.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:50:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2865.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:06] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2870.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:50:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2842.0>}
[ns_server:error] [2012-05-14 13:50:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:info] [2012-05-14 13:50:07] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:50:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2870.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2881.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:50:11] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2857.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2855.0>]
messages: [check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.36139>},connected},
check_started,
{'$gen_call',{<0.2861.0>,
{#Ref<0.0.0.36144>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2868.0>,#Ref<0.0.0.36251>},topkeys},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.36315>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.2877.0>,
{#Ref<0.0.0.36336>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2856.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7357
neighbours:
[error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2855.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2855.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2884.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2858.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:11] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2887.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:50:12] [ns_1@10.1.3.83:<0.1744.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:50:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:50:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2881.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:12] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2895.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:50:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2895.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:15] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2902.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:16] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:50:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2842.0>}
[ns_doctor:debug] [2012-05-14 13:50:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28611,864006}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13713416},
{processes,5254576},
{processes_used,5243072},
{system,8458840},
{atom,702357},
{atom_used,681123},
{binary,305696},
{code,5596508},
{ets,549180}]},
{system_stats,
[{cpu_utilization_rate,27.71385692846423},
{swap_total,9850605568},
{swap_used,720048128}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,776},
{memory_data,{4294500352,599257088,{<0.1620.0>,415388}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3696807936},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{772672,0}},
{context_switches,{115513,0}},
{garbage_collection,{18099,87193107,0}},
{io,{{input,9812485},{output,6869119}}},
{reductions,{23812751,291826}},
{run_queue,0},
{runtime,{28546,734}}]}]}]
[ns_server:error] [2012-05-14 13:50:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:info] [2012-05-14 13:50:17] [ns_1@10.1.3.83:<0.2842.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:50:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2902.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:18] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2911.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:50:19] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2818.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2927.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37061>} event handler: ok
[ns_server:debug] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2930.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37074>} event handler: ok
[menelaus:warn] [2012-05-14 13:50:20] [ns_1@10.1.3.83:<0.2818.0>:menelaus_web:loop:334] Server error during processing: ["web request failed",
{path,
"/pools/default/buckets/default/nodes/10.1.3.83:8091/stats"},
{type,error},
{what,function_clause},
{trace,
[{lists,map,
[#Fun,
undefined]},
{menelaus_stats,jsonify_hks,1},
{menelaus_stats,
'-handle_bucket_node_stats/4-fun-0-',7},
{menelaus_web_buckets,
checking_bucket_access,4},
{menelaus_web,
handle_abortable_get_request,2},
{menelaus_web_buckets,
checking_bucket_access,4},
{menelaus_web,loop,3},
{mochiweb_http,headers,5}]}]
[ns_server:warn] [2012-05-14 13:50:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2911.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:21] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2936.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:22] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2886.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2884.0>]
messages: [check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.36582>},connected},
check_started,
{'$gen_call',{<0.2896.0>,
{#Ref<0.0.0.36596>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.2904.0>,#Ref<0.0.0.36701>},topkeys},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.36752>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.2912.0>,
{#Ref<0.0.0.36803>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2885.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7353
neighbours:
[error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2884.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2937.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.2842.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4288
neighbours:
[error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2887.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2940.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2884.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[stats:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:<0.2817.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:50:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:50:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2936.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:24] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2951.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:debug] [2012-05-14 13:50:25] [ns_1@10.1.3.83:<0.2956.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37399>} event handler: ok
[ns_server:debug] [2012-05-14 13:50:25] [ns_1@10.1.3.83:<0.2958.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37408>} event handler: ok
[menelaus:warn] [2012-05-14 13:50:25] [ns_1@10.1.3.83:<0.2921.0>:menelaus_web:loop:334] Server error during processing: ["web request failed",
{path,
"/pools/default/buckets/default/nodes/10.1.3.83:8091/stats"},
{type,error},
{what,function_clause},
{trace,
[{lists,map,
[#Fun,
undefined]},
{menelaus_stats,jsonify_hks,1},
{menelaus_stats,
'-handle_bucket_node_stats/4-fun-0-',7},
{menelaus_web_buckets,
checking_bucket_access,4},
{menelaus_web,
handle_abortable_get_request,2},
{menelaus_web_buckets,
checking_bucket_access,4},
{menelaus_web,loop,3},
{mochiweb_http,headers,5}]}]
[ns_server:info] [2012-05-14 13:50:25] [ns_1@10.1.3.83:ns_log:ns_log:handle_cast:115] suppressing duplicate log menelaus_web:19("Server error during processing: [\"web request failed\",\n {path,\n \"/pools/default/buckets/default/nodes/10.1.3.83:8091/stats\"},\n {type,error},\n {what,function_clause},\n {trace,\n [{lists,map,\n [#Fun,\n undefined]},\n {menelaus_stats,jsonify_hks,1},\n {menelaus_stats,\n '-handle_bucket_node_stats/4-fun-0-',7},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,\n handle_abortable_get_request,2},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,loop,3},\n {mochiweb_http,headers,5}]}]") because it's been seen 1 times in the past 5.016006 secs (last seen 5.016006 secs ago
[ns_server:warn] [2012-05-14 13:50:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:50:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:50:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2951.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:27] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2970.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2970.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2974.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:<0.1744.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:50:32] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2939.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2937.0>]
messages: [check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.37302>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.2961.0>,#Ref<0.0.0.37454>},topkeys},
check_started,
{'$gen_call',{<0.2966.0>,
{#Ref<0.0.0.37481>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.37509>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.2938.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7350
neighbours:
[error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2937.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.2980.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2940.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2937.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:50:32] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2982.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:50:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:50:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2974.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.2990.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:50:33] [ns_1@10.1.3.83:<0.2993.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37814>} event handler: ok
[ns_server:debug] [2012-05-14 13:50:33] [ns_1@10.1.3.83:<0.2995.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.37823>} event handler: ok
[menelaus:warn] [2012-05-14 13:50:33] [ns_1@10.1.3.83:<0.2954.0>:menelaus_web:loop:334] Server error during processing: ["web request failed",
{path,
"/pools/default/buckets/default/nodes/10.1.3.83:8091/stats"},
{type,error},
{what,function_clause},
{trace,
[{lists,map,
[#Fun,
undefined]},
{menelaus_stats,jsonify_hks,1},
{menelaus_stats,
'-handle_bucket_node_stats/4-fun-0-',7},
{menelaus_web_buckets,
checking_bucket_access,4},
{menelaus_web,
handle_abortable_get_request,2},
{menelaus_web_buckets,
checking_bucket_access,4},
{menelaus_web,loop,3},
{mochiweb_http,headers,5}]}]
[ns_server:info] [2012-05-14 13:50:33] [ns_1@10.1.3.83:ns_log:ns_log:handle_cast:115] suppressing duplicate log menelaus_web:19("Server error during processing: [\"web request failed\",\n {path,\n \"/pools/default/buckets/default/nodes/10.1.3.83:8091/stats\"},\n {type,error},\n {what,function_clause},\n {trace,\n [{lists,map,\n [#Fun,\n undefined]},\n {menelaus_stats,jsonify_hks,1},\n {menelaus_stats,\n '-handle_bucket_node_stats/4-fun-0-',7},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,\n handle_abortable_get_request,2},\n {menelaus_web_buckets,\n checking_bucket_access,4},\n {menelaus_web,loop,3},\n {mochiweb_http,headers,5}]}]") because it's been seen 2 times in the past 13.110007 secs (last seen 8.094001 secs ago
[ns_server:warn] [2012-05-14 13:50:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.2990.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3002.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:50:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2964.0>}
[ns_server:error] [2012-05-14 13:50:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:info] [2012-05-14 13:50:38] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:50:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3002.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3014.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:50:40] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:50:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.2983.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.2980.0>]
messages: [{'$gen_call',{<0.231.0>,#Ref<0.0.0.37755>},connected},
check_started,check_started,
{'$gen_call',{<0.2986.0>,
{#Ref<0.0.0.37760>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3003.0>,#Ref<0.0.0.37932>},topkeys},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.37978>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3010.0>,
{#Ref<0.0.0.38002>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.2981.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7353
neighbours:
[error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.2980.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.2980.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3021.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.2982.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3024.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3014.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3028.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:50:45] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:50:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3028.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3038.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:50:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.2964.0>}
[ns_server:error] [2012-05-14 13:50:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2920.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:50:48] [ns_1@10.1.3.83:<0.2964.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:50:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3038.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3054.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:50:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:50:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:50:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3054.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:51] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3061.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3021.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3021.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3062.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3023.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3021.0>]
messages: [{'$gen_call',{<0.231.0>,#Ref<0.0.0.38254>},connected},
check_started,check_started,
{'$gen_call',{<0.3029.0>,
{#Ref<0.0.0.38278>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3039.0>,#Ref<0.0.0.38410>},topkeys},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.38456>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3055.0>,
{#Ref<0.0.0.38589>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.3022.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7353
neighbours:
[error_logger:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3024.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:50:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3065.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:50:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:50:53] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason shutdown
[ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:91] Stopping child for dead bucket: {{per_bucket_sup,"default"},
<0.3062.0>,supervisor,
[single_bucket_sup]}
[ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[]}]
[ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"
[ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:50:53] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:warn] [2012-05-14 13:50:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:50:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:50:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:50:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:51:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: shutdown_error
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3062.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3074.0>:ns_pubsub:do_subscribe_link:132] Parent process exited with reason normal
[ns_server:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:idle:373] Restarting moxi on nodes ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3074.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.38838>} event handler: ok
[error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3064.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3062.0>]
messages: [{'$gen_call',{<0.2964.0>,#Ref<0.0.0.38676>},
list_vbuckets_prevstate},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.38748>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.39057>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,check_started,
check_started,check_started,
{'EXIT',<0.3063.0>,shutdown}]
links: [<0.64.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7338
neighbours:
[error_logger:error] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3062.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[ns_server:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3102.0>:ns_port_sup:restart_port:92] restarting port: {moxi,"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}
[ns_server:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.1648.0>:ns_port_server:log:161] moxi<0.1648.0>: EOL on stdin. Exiting
[menelaus:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.1620.0>:menelaus_web:handle_streaming:707] menelaus_web streaming socket closed by client
[ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3104.0>:supervisor_cushion:init:43] starting ns_port_server with delay of 5000
[menelaus:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.2920.0>:menelaus_web_buckets:handle_bucket_delete:192] Deleted bucket "default"
[error_logger:info] [2012-05-14 13:51:02] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_port_sup}
started: [{pid,<0.3104.0>},
{name,
{moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]}},
{mfargs,
{supervisor_cushion,start_link,
[moxi,5000,ns_port_server,start_link,
[moxi,
"c:/Program Files/Couchbase/Server/bin/moxi",
["-Z",
"port_listen=11211,default_bucket_name=default,downstream_max=1024,downstream_conn_max=4,connect_max_errors=5,connect_retry_interval=30000,connect_timeout=400,auth_timeout=100,cycle=200,downstream_conn_queue_timeout=200,downstream_timeout=5000,wait_queue_timeout=200",
"-z",
"url=http://127.0.0.1:8091/pools/default/saslBucketsStreaming",
"-p","0","-Y","y","-O","stderr",[]],
[{env,
[{"EVENT_NOSELECT","1"},
{"MOXI_SASL_PLAIN_USR","Administrator"},
{"MOXI_SASL_PLAIN_PWD","password"}]},
use_stdio,exit_status,port_server_send_eol,
stderr_to_stdout,stream]]]}},
{restart_type,permanent},
{shutdown,10000},
{child_type,worker}]
[ns_server:debug] [2012-05-14 13:51:02] [ns_1@10.1.3.83:<0.3075.0>:menelaus_web:handle_streaming:673] Starting streaming for 127.0.0.1:61932 path /pools/default/saslBucketsStreaming
[ns_server:info] [2012-05-14 13:51:03] [ns_1@10.1.3.83:<0.3105.0>:ns_port_server:log:161] moxi<0.3105.0>: 2012-05-14 13:51:02: (cproxy_config.c.317) env: MOXI_SASL_PLAIN_USR (13)
moxi<0.3105.0>: 2012-05-14 13:51:02: (cproxy_config.c.326) env: MOXI_SASL_PLAIN_PWD (8)
[ns_doctor:debug] [2012-05-14 13:51:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28672,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,13993296},
{processes,5227196},
{processes_used,5213708},
{system,8766100},
{atom,728565},
{atom_used,713266},
{binary,448024},
{code,5718270},
{ets,565524}]},
{system_stats,
[{cpu_utilization_rate,25.4},
{swap_total,9850605568},
{swap_used,718110720}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,837},
{memory_data,{4294500352,597471232,{<0.1620.0>,415388}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3698405376},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{827688,0}},
{context_switches,{122991,0}},
{garbage_collection,{19152,92275520,0}},
{io,{{input,9961816},{output,7145454}}},
{reductions,{24890443,196517}},
{run_queue,0},
{runtime,{31093,453}}]}]}]
[ns_doctor:debug] [2012-05-14 13:52:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28732,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,13652992},
{processes,4862152},
{processes_used,4847512},
{system,8790840},
{atom,728565},
{atom_used,713266},
{binary,487136},
{code,5718270},
{ets,551852}]},
{system_stats,
[{cpu_utilization_rate,25.418645338665332},
{swap_total,9850605568},
{swap_used,717627392}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,897},
{memory_data,{4294500352,592855040,{<0.2920.0>,300780}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3700641792},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{887656,0}},
{context_switches,{129227,0}},
{garbage_collection,{20095,97210473,0}},
{io,{{input,10001792},{output,7354126}}},
{reductions,{25936110,147304}},
{run_queue,0},
{runtime,{32796,406}}]}]}]
[ns_doctor:debug] [2012-05-14 13:53:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28792,645005}},
{active_buckets,[]},
{ready_buckets,[]},
{replication,[]},
{memory,
[{total,13901968},
{processes,5154796},
{processes_used,5140652},
{system,8747172},
{atom,728565},
{atom_used,713266},
{binary,426496},
{code,5718270},
{ets,568452}]},
{system_stats,
[{cpu_utilization_rate,25.381345336334082},
{swap_total,9850605568},
{swap_used,723779584}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,957},
{memory_data,{4294500352,593780736,{<0.3152.0>,300780}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3688812544},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{947656,0}},
{context_switches,{135133,0}},
{garbage_collection,{21102,101126711,0}},
{io,{{input,10285014},{output,7756644}}},
{reductions,{26936431,147602}},
{run_queue,0},
{runtime,{34265,328}}]}]}]
[menelaus:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3318.0>:menelaus_web_buckets:do_bucket_create:262] Created bucket "default" of type: membase
[{num_replicas,1},{ram_quota,3435134976},{auth_type,sasl}]
[ns_server:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[[{map,[]},
{fastForwardMap,[]},
{sasl_password,[]},
{num_replicas,1},
{ram_quota,3435134976},
{auth_type,sasl},
{type,membase},
{num_vbuckets,1024},
{servers,[]}]]}]
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_isasl_sync:ns_config_isasl_sync:writeSASLConf:129] Writing isasl passwd file: "c:/Program Files/Couchbase/Server/var/lib/couchbase/data/isasl.pw"
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_bucket_worker:ns_bucket_sup:update_childs:84] Starting new child: {{per_bucket_sup,"default"},
{single_bucket_sup,start_link,["default"]},
permanent,infinity,supervisor,
[single_bucket_sup]}
[error_logger:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3488.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_log:ns_config_log:log_common:111] config change:
buckets ->
[{configs,[{"default",
[{map,[]},
{fastForwardMap,[]},
{sasl_password,[]},
{num_replicas,1},
{ram_quota,3435134976},
{auth_type,sasl},
{type,membase},
{num_vbuckets,1024},
{servers,['ns_1@10.1.3.83']}]}]}]
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_events:ns_node_disco_conf_events:handle_event:56] ns_node_disco_conf_events config all
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:201] Pushing config
[ns_server:debug] [2012-05-14 13:53:28] [ns_1@10.1.3.83:ns_config_rep:ns_config_rep:handle_info:203] Pushing config done
[stats:error] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3318.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3105.0>:ns_port_server:log:161] moxi<0.3105.0>: 2012-05-14 13:53:30: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({
moxi<0.3105.0>: "name": "default",
moxi<0.3105.0>: "nodeLocator": "vbucket",
moxi<0.3105.0>: "saslPassword": "",
moxi<0.3105.0>: "nodes": [{
moxi<0.3105.0>: "replication": "undefined",
moxi<0.3105.0>: "clusterMembership": "active",
moxi<0.3105.0>: "status": "warmup",
moxi<0.3105.0>: "hostname": "10.1.3.83:8091",
moxi<0.3105.0>: "clusterCompatibility": 1,
moxi<0.3105.0>: "version": "1.8.1-815-rel-enterprise",
moxi<0.3105.0>: "os": "windows",
moxi<0.3105.0>: "ports": {
moxi<0.3105.0>: "proxy": 11211,
moxi<0.3105.0>: "direct": 11210
moxi<0.3105.0>: }
moxi<0.3105.0>: }],
moxi<0.3105.0>: "vBucketServerMap": {
moxi<0.3105.0>: "hashAlgorithm": "CRC",
moxi<0.3105.0>: "numReplicas": 1,
moxi<0.3105.0>: "serverList": ["10.1.3.83:11210"],
moxi<0.3105.0>: "vBucketMap": []
moxi<0.3105.0>: }
moxi<0.3105.0>: })
[stats:error] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3313.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:53:28] [ns_1@10.1.3.83:<0.3313.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:53:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:53:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3061.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3509.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:53:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:53:31] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:53:31] [ns_1@10.1.3.83:<0.3313.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:error] [2012-05-14 13:53:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:53:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:53:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3509.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:33] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3529.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:53:34] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:53:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:53:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3529.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:36] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3534.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:53:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:53:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3486.0>}
[ns_server:error] [2012-05-14 13:53:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:info] [2012-05-14 13:53:37] [ns_1@10.1.3.83:<0.3105.0>:ns_port_server:log:161] moxi<0.3105.0>: 2012-05-14 13:53:39: (agent_config.c.705) ERROR: bad JSON configuration from http://127.0.0.1:8091/pools/default/saslBucketsStreaming: Number of buckets must be a power of two > 0 and <= MAX_BUCKETS ({
moxi<0.3105.0>: "name": "default",
moxi<0.3105.0>: "nodeLocator": "vbucket",
moxi<0.3105.0>: "saslPassword": "",
moxi<0.3105.0>: "nodes": [{
moxi<0.3105.0>: "replication": 1,
moxi<0.3105.0>: "clusterMembership": "active",
moxi<0.3105.0>: "status": "warmup",
moxi<0.3105.0>: "hostname": "10.1.3.83:8091",
moxi<0.3105.0>: "clusterCompatibility": 1,
moxi<0.3105.0>: "version": "1.8.1-815-rel-enterprise",
moxi<0.3105.0>: "os": "windows",
moxi<0.3105.0>: "ports": {
moxi<0.3105.0>: "proxy": 11211,
moxi<0.3105.0>: "direct": 11210
moxi<0.3105.0>: }
moxi<0.3105.0>: }],
moxi<0.3105.0>: "vBucketServerMap": {
moxi<0.3105.0>: "hashAlgorithm": "CRC",
moxi<0.3105.0>: "numReplicas": 1,
moxi<0.3105.0>: "serverList": ["10.1.3.83:11210"],
moxi<0.3105.0>: "vBucketMap": []
moxi<0.3105.0>: }
moxi<0.3105.0>: })
[ns_server:info] [2012-05-14 13:53:38] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3490.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3488.0>]
messages: [check_started,
{'$gen_call',{<0.3505.0>,
{#Ref<0.0.0.44696>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.3525.0>,#Ref<0.0.0.45190>},topkeys},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.45209>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.3530.0>,
{#Ref<0.0.0.45273>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.45397>},connected},
check_started,check_started]
links: [<0.64.0>,<0.3489.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3065.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3541.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3488.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3542.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:53:38] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3488.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[ns_server:warn] [2012-05-14 13:53:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:53:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3534.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3551.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:53:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:53:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:53:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3551.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:42] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3555.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:53:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:53:44] [ns_1@10.1.3.83:<0.3486.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:53:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:53:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3555.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:45] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3563.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:53:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3486.0>}
[ns_server:warn] [2012-05-14 13:53:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:53:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:<0.3318.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:53:48] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3544.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3542.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3547.0>,
{#Ref<0.0.0.45513>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.45623>},connected},
check_started,check_started,
{'$gen_call',{<0.3556.0>,#Ref<0.0.0.45646>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3559.0>,
{#Ref<0.0.0.45689>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.45793>},connected},
check_started,check_started]
links: [<0.64.0>,<0.3543.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.3486.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4286
neighbours:
[error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3542.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3542.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3571.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3541.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3573.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3563.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:48] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3580.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:53:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:53:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:53:51] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:53:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3580.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3590.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:53:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:53:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:53:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3590.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3595.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:53:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:53:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:53:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3595.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3606.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3574.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3571.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.46126>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.3593.0>,#Ref<0.0.0.46148>},topkeys},
check_started,check_started,check_started,check_started,
check_started,check_started,check_started,
{'$gen_call',{<0.3602.0>,
{#Ref<0.0.0.46257>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.46281>},connected},
check_started,check_started,check_started]
links: [<0.64.0>,<0.3572.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7347
neighbours:
[error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3571.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3607.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3571.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3573.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:53:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3610.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:53:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3606.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3617.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:54:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:54:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3617.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3624.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:54:04] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:54:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3624.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3630.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:54:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3600.0>}
[ns_server:error] [2012-05-14 13:54:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:54:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:54:09] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3609.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3607.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3615.0>,
{#Ref<0.0.0.46392>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.46477>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.3622.0>,#Ref<0.0.0.46510>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3628.0>,
{#Ref<0.0.0.46581>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.46665>},connected},
check_started,check_started,check_started]
links: [<0.64.0>,<0.3608.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7355
neighbours:
[error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3607.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3607.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3637.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3610.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3640.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3630.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3646.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:54:11] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:54:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:54:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:54:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3646.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3655.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:54:15] [ns_1@10.1.3.83:<0.3600.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3655.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3662.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:16] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:54:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3600.0>}
[ns_doctor:debug] [2012-05-14 13:54:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28849,67009}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,14109000},
{processes,5288192},
{processes_used,5276048},
{system,8820808},
{atom,728565},
{atom_used,713266},
{binary,513080},
{code,5718270},
{ets,554388}]},
{system_stats,
[{cpu_utilization_rate,27.411167512690355},
{swap_total,9850605568},
{swap_used,724307968}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,1014},
{memory_data,{4294500352,605745152,{<0.3314.0>,300780}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3688005632},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{1012672,0}},
{context_switches,{143434,0}},
{garbage_collection,{22390,107611232,0}},
{io,{{input,10326587},{output,8138622}}},
{reductions,{28413884,285246}},
{run_queue,0},
{runtime,{36546,812}}]}]}]
[ns_server:error] [2012-05-14 13:54:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:54:18] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3662.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3671.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:19] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3639.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3637.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3647.0>,
{#Ref<0.0.0.46776>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.46931>},connected},
check_started,check_started,check_started,
{'$gen_call',{<0.3658.0>,#Ref<0.0.0.46970>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3663.0>,
{#Ref<0.0.0.47026>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.47100>},connected},
check_started,check_started,check_started]
links: [<0.64.0>,<0.3638.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3637.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3672.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3640.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.3600.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4288
neighbours:
[error_logger:info] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3674.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3637.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[ns_server:warn] [2012-05-14 13:54:20] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3671.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3685.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:22] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:54:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:54:24] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3685.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3690.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:26] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:54:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:54:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3690.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3701.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:28] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:54:29] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3672.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3672.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3702.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3675.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3672.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.47309>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3688.0>,#Ref<0.0.0.47339>},topkeys},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.3697.0>,
{#Ref<0.0.0.47436>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.47464>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.3673.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7347
neighbours:
[error_logger:error] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3674.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:29] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3705.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:30] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3701.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3712.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:54:31] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:54:32] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:54:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[stats:error] [2012-05-14 13:54:33] [ns_1@10.1.3.83:<0.3514.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3712.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3727.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:34] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:54:35] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:54:36] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:debug] [2012-05-14 13:54:36] [ns_1@10.1.3.83:<0.3733.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.47976>} event handler: ok
[stats:error] [2012-05-14 13:54:36] [ns_1@10.1.3.83:<0.3514.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3727.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3740.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:54:37] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3695.0>}
[ns_server:error] [2012-05-14 13:54:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:54:38] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:debug] [2012-05-14 13:54:38] [ns_1@10.1.3.83:<0.3747.0>:ns_pubsub:do_subscribe_link:145] Deleting {ns_pubsub,#Ref<0.0.0.48156>} event handler: ok
[error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3702.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3704.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3702.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3710.0>,
{#Ref<0.0.0.47585>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.47751>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3728.0>,#Ref<0.0.0.47902>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3731.0>,
{#Ref<0.0.0.47952>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.48125>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.3703.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3702.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3749.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3705.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:39] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3751.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3625.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[stats:error] [2012-05-14 13:54:39] [ns_1@10.1.3.83:<0.3514.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3740.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3770.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:40] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:54:42] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:54:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:54:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3770.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3775.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:44] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:54:45] [ns_1@10.1.3.83:<0.3695.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3775.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3782.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:46] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:54:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3695.0>}
[ns_server:error] [2012-05-14 13:54:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:54:48] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3782.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3789.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:49] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {noproc,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3752.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3749.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3771.0>,
{#Ref<0.0.0.48452>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.48527>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3778.0>,#Ref<0.0.0.48570>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3783.0>,
{#Ref<0.0.0.48627>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.48690>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.3750.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7351
neighbours:
[error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3749.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.3695.0>
registered_name: []
exception exit: {noproc,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4286
neighbours:
[error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3749.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3792.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3751.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3795.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:50] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:54:51] [ns_1@10.1.3.83:<0.3511.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3789.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3806.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:54:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:54:52] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:54:54] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:54:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3806.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3811.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:56] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:54:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:54:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3811.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3822.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:54:58] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:54:59] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3794.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3792.0>]
messages: [check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.48942>},connected},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3809.0>,#Ref<0.0.0.48995>},topkeys},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.3818.0>,
{#Ref<0.0.0.49077>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.49105>},connected},
check_started,check_started,check_started,check_started]
links: [<0.64.0>,<0.3793.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7349
neighbours:
[error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3792.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3792.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3829.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3795.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:54:59] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3832.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:00] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3822.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3837.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:55:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:02] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3837.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3843.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:04] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:05] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:warn] [2012-05-14 13:55:06] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3843.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3849.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:55:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3816.0>}
[ns_server:error] [2012-05-14 13:55:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:08] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:09] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3831.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3829.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3835.0>,
{#Ref<0.0.0.49315>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.49386>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.3844.0>,#Ref<0.0.0.49443>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3847.0>,
{#Ref<0.0.0.49498>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.49567>},connected},
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.3830.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7357
neighbours:
[error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3829.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3829.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3858.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3832.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:09] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3859.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:55:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3849.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3865.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:10] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:55:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:12] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3865.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3870.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:14] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:15] [ns_1@10.1.3.83:<0.3816.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:55:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3870.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3877.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:17] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:17] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3816.0>}
[ns_doctor:debug] [2012-05-14 13:55:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28909,927007}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13765264},
{processes,4949160},
{processes_used,4936000},
{system,8816104},
{atom,728565},
{atom_used,713266},
{binary,493592},
{code,5718270},
{ets,569908}]},
{system_stats,
[{cpu_utilization_rate,27.375},
{swap_total,9850605568},
{swap_used,725495808}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,1074},
{memory_data,{4294500352,606453760,{<0.3511.0>,300780}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3687280640},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{1072672,0}},
{context_switches,{151010,0}},
{garbage_collection,{23290,112155175,0}},
{io,{{input,10335561},{output,8427342}}},
{reductions,{29434667,265933}},
{run_queue,0},
{runtime,{38921,937}}]}]}]
[ns_server:error] [2012-05-14 13:55:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:19] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3877.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:19] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3899.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[stats:error] [2012-05-14 13:55:19] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3858.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3858.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3906.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3861.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3858.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3866.0>,
{#Ref<0.0.0.49690>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.49748>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.3873.0>,#Ref<0.0.0.49805>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3879.0>,
{#Ref<0.0.0.49868>,'ns_1@10.1.3.83'}},
connected},
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.50010>},connected},
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.3860.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3859.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:20] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3910.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:21] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3899.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:22] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3917.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:55:22] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:23] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:55:25] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3917.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:25] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3922.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:27] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:27] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3816.0>}
[ns_server:error] [2012-05-14 13:55:27] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:55:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3922.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:28] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3930.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:29] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:30] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3906.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3906.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3933.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3908.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3906.0>]
messages: [{'$gen_call',{<0.3816.0>,#Ref<0.0.0.50184>},
list_vbuckets_prevstate},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.50311>},connected},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.3920.0>,#Ref<0.0.0.50363>},topkeys},
check_started,check_started,check_started,check_started,
check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.50459>},connected},
check_started,check_started,check_started,check_started,
check_started]
links: [<0.64.0>,<0.3907.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7349
neighbours:
[error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.3816.0>
registered_name: []
exception exit: {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4291
neighbours:
[error_logger:error] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3910.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:30] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3936.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:31] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3930.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:31] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3940.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:55:32] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:33] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3940.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:34] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3946.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:35] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:55:37] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3946.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:37] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3951.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:55:37] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:39] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:55:39] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3935.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3933.0>]
messages: [check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.50659>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.3947.0>,#Ref<0.0.0.50716>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3956.0>,
{#Ref<0.0.0.50797>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.50832>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.3934.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7349
neighbours:
[error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3933.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3933.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3967.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3936.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3970.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3951.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:40] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3974.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:41] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:55:42] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:55:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3974.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:43] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3979.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:43] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:55:45] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:46] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:55:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3979.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:46] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3986.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:47] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:47] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3954.0>}
[ns_server:error] [2012-05-14 13:55:47] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:55:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3986.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:49] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3993.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:49] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:55:50] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3967.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3967.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.3996.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3969.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3967.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.3975.0>,
{#Ref<0.0.0.51040>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.51098>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.3982.0>,#Ref<0.0.0.51165>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.3987.0>,
{#Ref<0.0.0.51222>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.51270>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.3968.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3970.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:50] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.3999.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:51] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.3993.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:52] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4008.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:error] [2012-05-14 13:55:52] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:55:53] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:55:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4008.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:55] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4012.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:55] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:info] [2012-05-14 13:55:56] [ns_1@10.1.3.83:<0.3954.0>:ns_janitor:wait_for_memcached:286] Waiting for "default" on ['ns_1@10.1.3.83']
[ns_server:info] [2012-05-14 13:55:57] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3954.0>}
[ns_server:warn] [2012-05-14 13:55:57] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:55:57] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:55:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4012.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:55:58] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4022.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:55:59] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[stats:error] [2012-05-14 13:55:59] [ns_1@10.1.3.83:<0.3732.0>:stats_reader:log_bad_responses:191] Some nodes didn't respond: ['ns_1@10.1.3.83']
[error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.3996.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.3996.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.4029.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.3998.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.3996.0>]
messages: [check_started,check_started,
{'$gen_call',{<0.4004.0>,
{#Ref<0.0.0.51408>,'ns_1@10.1.3.83'}},
connected},
check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.51471>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.4013.0>,#Ref<0.0.0.51542>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.4018.0>,
{#Ref<0.0.0.51588>,'ns_1@10.1.3.83'}},
connected},
{'$gen_call',{<0.231.0>,#Ref<0.0.0.51629>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.3997.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7352
neighbours:
[error_logger:error] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.3999.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:56:00] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4032.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:56:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4022.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:56:01] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4036.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:56:01] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:56:02] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:56:03] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:56:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4036.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:56:04] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4042.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:56:05] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:56:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4042.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:56:07] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4047.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:info] [2012-05-14 13:56:07] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state,
["default"],
<0.3954.0>}
[ns_server:error] [2012-05-14 13:56:07] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[ns_server:warn] [2012-05-14 13:56:07] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:56:09] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4047.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4057.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_memcached:init/1
pid: <0.4031.0>
registered_name: []
exception exit: couldnt_connect_to_memcached
in function gen_server:init_it/6
ancestors: ['single_bucket_sup-default',<0.4029.0>]
messages: [{'$gen_call',{<0.3954.0>,#Ref<0.0.0.51788>},
list_vbuckets_prevstate},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.51894>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started,
{'$gen_call',{<0.4045.0>,#Ref<0.0.0.51970>},topkeys},
check_started,check_started,check_started,check_started,
{'$gen_call',{<0.231.0>,#Ref<0.0.0.52062>},connected},
check_started,check_started,check_started,check_started,
check_started,check_started]
links: [<0.64.0>,<0.4030.0>]
dictionary: []
trap_exit: true
status: running
heap_size: 28657
stack_size: 24
reductions: 7355
neighbours:
[error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {couldnt_connect_to_memcached,
{gen_server,call,['ns_memcached-default',topkeys,30000]}}
Offender: [{pid,<0.4032.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,'single_bucket_sup-default'}
Context: start_error
Reason: couldnt_connect_to_memcached
Offender: [{pid,undefined},
{name,{ns_memcached,"default"}},
{mfargs,{ns_memcached,start_link,["default"]}},
{restart_type,permanent},
{shutdown,86400000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:56:10] [ns_1@10.1.3.83:<0.218.0>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "default" with reason {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default',
'ns_1@10.1.3.83'},
list_vbuckets_prevstate,
30000]}}
[error_logger:info] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4058.0>},
{name,hot_keys_keeper},
{mfargs,{hot_keys_keeper,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================CRASH REPORT=========================
crasher:
initial call: ns_janitor:cleanup/2
pid: <0.3954.0>
registered_name: []
exception exit: {couldnt_connect_to_memcached,
{gen_server,call,
[{'ns_memcached-default','ns_1@10.1.3.83'},
list_vbuckets_prevstate,30000]}}
in function gen_server:call/3
in call from ns_janitor:'-read_existing_map/4-lc$^0/1-0-'/2
in call from ns_janitor:read_existing_map/4
in call from ns_janitor:do_cleanup/3
ancestors: [<0.218.0>,mb_master_sup,mb_master,ns_server_sup,
ns_server_cluster_sup,<0.67.0>]
messages: []
links: [<0.218.0>]
dictionary: []
trap_exit: false
status: running
heap_size: 28657
stack_size: 24
reductions: 4289
neighbours:
[error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,ns_bucket_sup}
Context: child_terminated
Reason: {{badmatch,{error,shutdown}},
[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
Offender: [{pid,<0.4029.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:info] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,ns_bucket_sup}
started: [{pid,<0.4059.0>},
{name,{per_bucket_sup,"default"}},
{mfargs,{single_bucket_sup,start_link,["default"]}},
{restart_type,permanent},
{shutdown,infinity},
{child_type,supervisor}]
[error_logger:error] [2012-05-14 13:56:10] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_msg:76] Error in process <0.4029.0> on node 'ns_1@10.1.3.83' with exit value: {{badmatch,{error,shutdown}},[{single_bucket_sup,'-start_link/1-fun-0-',2}]}
[ns_server:warn] [2012-05-14 13:56:11] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:error] [2012-05-14 13:56:12] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}
[error_logger:error] [2012-05-14 13:56:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4057.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:56:13] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4067.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_server:warn] [2012-05-14 13:56:13] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[ns_server:warn] [2012-05-14 13:56:15] [ns_1@10.1.3.83:'ns_memcached-default':ns_memcached:connect:576] Unable to connect: {error,{badmatch,{error,econnrefused}}}, retrying.
[error_logger:error] [2012-05-14 13:56:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================SUPERVISOR REPORT=========================
Supervisor: {local,menelaus_sup}
Context: child_terminated
Reason: {noproc,
{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute,1}]}}
Offender: [{pid,<0.4067.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[error_logger:info] [2012-05-14 13:56:16] [ns_1@10.1.3.83:error_logger:ale_error_logger_handler:log_report:72]
=========================PROGRESS REPORT=========================
supervisor: {local,menelaus_sup}
started: [{pid,<0.4075.0>},
{name,menelaus_web_alerts_srv},
{mfargs,{menelaus_web_alerts_srv,start_link,[]}},
{restart_type,permanent},
{shutdown,5000},
{child_type,worker}]
[ns_doctor:debug] [2012-05-14 13:56:17] [ns_1@10.1.3.83:ns_doctor:ns_doctor:handle_info:93] Current node statuses:
[{'ns_1@10.1.3.83',
[{last_heard,{1337,28970,817013}},
{active_buckets,["default"]},
{ready_buckets,[]},
{replication,[{"default",1.0}]},
{memory,
[{total,13687056},
{processes,4852820},
{processes_used,4839940},
{system,8834236},
{atom,728565},
{atom_used,713266},
{binary,494184},
{code,5718270},
{ets,587668}]},
{system_stats,
[{cpu_utilization_rate,28.125},
{swap_total,9850605568},
{swap_used,727543808}]},
{interesting_stats,[]},
{cluster_compatibility_version,1},
{version,
[{ale,"8cffe61"},
{os_mon,"2.2.5"},
{mnesia,"4.4.17"},
{kernel,"2.14.3"},
{sasl,"2.1.9.3"},
{ns_server,"1.8.1-815-rel-enterprise"},
{stdlib,"1.17.3"}]},
{system_arch,"windows"},
{wall_clock,1135},
{memory_data,{4294500352,607203328,{<0.231.0>,300660}}},
{disk_data,[{"C:\\",62810108,36}]},
{system_memory_data,
[{total_memory,4294500352},
{free_memory,3686322176},
{system_total_memory,4294500352}]},
{statistics,
[{wall_clock,{1132672,0}},
{context_switches,{158050,0}},
{garbage_collection,{24210,115452087,0}},
{io,{{input,10589364},{output,8875088}}},
{reductions,{30421724,250264}},
{run_queue,0},
{runtime,{41609,656}}]}]}]
[ns_server:error] [2012-05-14 13:56:17] [ns_1@10.1.3.83:ns_heart:ns_heart:current_status:163] Failed to get stats for bucket: "default":
{'EXIT',{noproc,{gen_server,call,
[{'stats_reader-default','ns_1@10.1.3.83'},
{latest,minute}]}}}